Merge branch 'master' into pr-style-v2-resizers

This commit is contained in:
James Allen
2017-12-04 10:01:45 +00:00
29 changed files with 11402 additions and 7337 deletions

View File

@@ -73,3 +73,4 @@ Gemfile.lock
app/views/external
/modules/
docker-shared.yml

View File

@@ -196,6 +196,7 @@ module.exports = (grunt) ->
"mathjax": "/js/libs/mathjax/MathJax.js?config=TeX-AMS_HTML"
"pdfjs-dist/build/pdf": "libs/#{PackageVersions.lib('pdfjs')}/pdf"
"ace": "#{PackageVersions.lib('ace')}"
"fineuploader": "libs/#{PackageVersions.lib('fineuploader')}"
shim:
"pdfjs-dist/build/pdf":
deps: ["libs/#{PackageVersions.lib('pdfjs')}/compatibility"]

View File

@@ -42,7 +42,7 @@ pipeline {
checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/tpr-webmodule'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/tpr-webmodule.git ']]])
checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/learn-wiki'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@bitbucket.org:sharelatex/learn-wiki-web-module.git']]])
checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/templates'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/templates-webmodule.git']]])
checkout([$class: 'GitSCM', branches: [[name: '*/sk-unlisted-projects']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/track-changes'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/track-changes-web-module.git']]])
checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/track-changes'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/track-changes-web-module.git']]])
checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/overleaf-integration'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/overleaf-integration-web-module.git']]])
checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/overleaf-account-merge'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/overleaf-account-merge.git']]])
}
@@ -70,6 +70,19 @@ pipeline {
sh 'ls -l node_modules/.bin'
}
}
stage('Unit Tests') {
steps {
sh 'make clean install' // Removes js files, so do before compile
sh 'make test_unit MOCHA_ARGS="--reporter=tap"'
}
}
stage('Acceptance Tests') {
steps {
sh 'make test_acceptance MOCHA_ARGS="--reporter=tap"'
}
}
stage('Compile') {
agent {
@@ -109,30 +122,6 @@ pipeline {
}
}
stage('Unit Test') {
agent {
docker {
image 'node:6.9.5'
reuseNode true
}
}
steps {
sh 'env NODE_ENV=development ./node_modules/.bin/grunt mochaTest:unit --reporter=tap'
}
}
stage('Acceptance Tests') {
steps {
// This tagged relase of the acceptance test runner is a temporary fix
// to get the acceptance tests working before we move to a
// docker-compose workflow. See:
// https://github.com/sharelatex/web-sharelatex-internal/pull/148
sh 'docker pull sharelatex/sl-acceptance-test-runner:node-6.9-mongo-3.4'
sh 'docker run --rm -v $(pwd):/app --env SHARELATEX_ALLOW_PUBLIC_ACCESS=true sharelatex/sl-acceptance-test-runner:node-6.9-mongo-3.4 || (cat forever/app.log && false)'
}
}
stage('Package') {
steps {
sh 'rm -rf ./node_modules/grunt*'

77
services/web/Makefile Normal file
View File

@@ -0,0 +1,77 @@
DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
NPM := docker-compose ${DOCKER_COMPOSE_FLAGS} run --rm npm npm -q
BUILD_NUMBER ?= local
BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
PROJECT_NAME = web
all: install test
@echo "Run:"
@echo " make install to set up the project dependencies (in docker)"
@echo " make test to run all the tests for the project (in docker)"
add: docker-shared.yml
$(NPM) install --save ${P}
add_dev: docker-shared.yml
$(NPM) install --save-dev ${P}
install: docker-shared.yml
$(NPM) install
clean:
rm -f app.js
rm -rf app/js
rm -rf test/unit/js
rm -rf test/acceptance/js
for dir in modules/*; \
do \
rm -f $$dir/index.js; \
rm -rf $$dir/app/js; \
rm -rf $$dir/test/unit/js; \
rm -rf $$dir/test/acceptance/js; \
done
# Regenerate docker-shared.yml - not stictly a 'clean',
# but lets `make clean install` work nicely
bin/generate_volumes_file
# Deletes node_modules volume
docker-compose down --volumes
# Need regenerating if you change the web modules you have installed
docker-shared.yml:
bin/generate_volumes_file
test: test_unit test_acceptance
test_unit: docker-shared.yml
docker-compose ${DOCKER_COMPOSE_FLAGS} run --rm test_unit npm -q run test:unit -- ${MOCHA_ARGS}
test_acceptance: test_acceptance_app test_acceptance_modules
test_acceptance_app: test_acceptance_app_start_service test_acceptance_app_run test_acceptance_app_stop_service
test_acceptance_app_start_service: test_acceptance_app_stop_service docker-shared.yml
docker-compose ${DOCKER_COMPOSE_FLAGS} up -d test_acceptance
test_acceptance_app_stop_service: docker-shared.yml
docker-compose ${DOCKER_COMPOSE_FLAGS} stop test_acceptance redis mongo
test_acceptance_app_run: docker-shared.yml
docker-compose ${DOCKER_COMPOSE_FLAGS} exec -T test_acceptance npm -q run test:acceptance -- ${MOCHA_ARGS}
test_acceptance_modules: docker-shared.yml
# Break and error on any module failure
set -e; \
for dir in modules/*; \
do \
if [ -e $$dir/Makefile ]; then \
(make test_acceptance_module MODULE=$$dir) \
fi \
done
test_acceptance_module: docker-shared.yml
cd $(MODULE) && make test_acceptance
.PHONY:
all add install update test test_unit test_acceptance \
test_acceptance_start_service test_acceptance_stop_service \
test_acceptance_run

View File

@@ -17,6 +17,69 @@ web-sharelatex uses [Grunt](http://gruntjs.com/) to build its front-end related
Image processing tasks are commented out in the gruntfile and the needed packages aren't presently in the project's `package.json`. If the images need to be processed again (minified and sprited), start by fetching the packages (`npm install grunt-contrib-imagemin grunt-sprity`), then *decomment* the tasks in `Gruntfile.coffee`. After this, the tasks can be called (explicitly, via `grunt imagemin` and `grunt sprity`).
New Docker-based build process
------------------------------
Note that the Grunt workflow from above should still work, but we are transitioning to a
Docker based testing workflow, which is documented below:
### Running the app
The app runs natively using npm and Node on the local system:
```
$ npm install
$ npm run start
```
*Ideally the app would run in Docker like the tests below, but with host networking not supported in OS X, we need to run it natively until all services are Dockerised.*
### Unit Tests
The test suites run in Docker.
Unit tests can be run in the `test_unit` container defined in `docker-compose.tests.yml`.
The makefile contains a short cut to run these:
```
make install # Only needs running once, or when npm packages are updated
make unit_test
```
During development it is often useful to only run a subset of tests, which can be configured with arguments to the mocha CLI:
```
make unit_test MOCHA_ARGS='--grep=AuthorizationManager'
```
### Acceptance Tests
Acceptance tests are run against a live service, which runs in the `acceptance_test` container defined in `docker-compose.tests.yml`.
To run the tests out-of-the-box, the makefile defines:
```
make install # Only needs running once, or when npm packages are updated
make acceptance_test
```
However, during development it is often useful to leave the service running for rapid iteration on the acceptance tests. This can be done with:
```
make acceptance_test_start_service
make acceptance_test_run # Run as many times as needed during development
make acceptance_test_stop_service
```
`make acceptance_test` just runs these three commands in sequence.
During development it is often useful to only run a subset of tests, which can be configured with arguments to the mocha CLI:
```
make acceptance_test_run MOCHA_ARGS='--grep=AuthorizationManager'
```
Unit test status
----------------

View File

@@ -251,7 +251,7 @@ module.exports = ProjectController =
# Extract data from user's ObjectId
timestamp = parseInt(user_id.toString().substring(0, 8), 16)
rolloutPercentage = 10 # Percentage of users to roll out to
rolloutPercentage = 20 # Percentage of users to roll out to
if !ProjectController._isInPercentageRollout('autocompile', user_id, rolloutPercentage)
# Don't show if user is not part of roll out
return cb(null, { enabled: false, showOnboarding: false })

View File

@@ -24,6 +24,7 @@ jsPath =
ace = PackageVersions.lib('ace')
pdfjs = PackageVersions.lib('pdfjs')
fineuploader = PackageVersions.lib('fineuploader')
getFileContent = (filePath)->
filePath = Path.join __dirname, "../../../", "public#{filePath}"
@@ -37,6 +38,7 @@ getFileContent = (filePath)->
logger.log "Generating file fingerprints..."
pathList = [
["#{jsPath}libs/#{fineuploader}.js"]
["#{jsPath}libs/require.js"]
["#{jsPath}ide.js"]
["#{jsPath}main.js"]

View File

@@ -2,6 +2,7 @@ version = {
"pdfjs": "1.7.225"
"moment": "2.9.0"
"ace": "1.2.5"
"fineuploader": "5.15.4"
}
module.exports = {

View File

@@ -132,7 +132,8 @@ html(itemscope, itemtype='http://schema.org/Product')
// minimal requirejs configuration (can be extended/overridden)
window.requirejs = {
"paths" : {
"moment": "libs/#{lib('moment')}"
"moment": "libs/#{lib('moment')}",
"fineuploader": "libs/#{lib('fineuploader')}"
},
"urlArgs": "fingerprint=#{fingerprint(jsPath + 'main.js')}-#{fingerprint(jsPath + 'libs.js')}",
"config":{

View File

@@ -130,7 +130,8 @@ block requirejs
"moment": "libs/#{lib('moment')}",
"pdfjs-dist/build/pdf": "libs/#{lib('pdfjs')}/pdf",
"pdfjs-dist/build/pdf.worker": "#{pdfWorkerPath}",
"ace": "#{lib('ace')}"
"ace": "#{lib('ace')}",
"fineuploader": "libs/#{lib('fineuploader')}"
},
"urlArgs" : "fingerprint=#{fingerprint(jsPath + 'ide.js')}-#{fingerprint(jsPath + 'libs.js')}",
"waitSeconds": 0,

View File

@@ -0,0 +1,4 @@
#!/bin/bash
set -e;
MOCHA="node_modules/.bin/mocha --recursive --reporter spec --timeout 15000"
$MOCHA "$@"

View File

@@ -0,0 +1,16 @@
#!/bin/bash
set -e;
COFFEE=node_modules/.bin/coffee
echo Compiling test/acceptance/coffee;
$COFFEE -o test/acceptance/js -c test/acceptance/coffee;
for dir in modules/*;
do
if [ -d $dir/test/acceptance ]; then
echo Compiling $dir/test/acceptance/coffee;
$COFFEE -o $dir/test/acceptance/js -c $dir/test/acceptance/coffee;
fi
done

23
services/web/bin/compile_app Executable file
View File

@@ -0,0 +1,23 @@
#!/bin/bash
set -e;
COFFEE=node_modules/.bin/coffee
echo Compiling app.coffee;
$COFFEE -c app.coffee;
echo Compiling app/coffee;
$COFFEE -o app/js -c app/coffee;
for dir in modules/*;
do
if [ -d $dir/app/coffee ]; then
echo Compiling $dir/app/coffee;
$COFFEE -o $dir/app/js -c $dir/app/coffee;
fi
if [ -e $dir/index.coffee ]; then
echo Compiling $dir/index.coffee;
$COFFEE -c $dir/index.coffee;
fi
done

View File

@@ -0,0 +1,15 @@
#!/bin/bash
set -e;
COFFEE=node_modules/.bin/coffee
echo Compiling test/unit/coffee;
$COFFEE -o test/unit/js -c test/unit/coffee;
for dir in modules/*;
do
if [ -d $dir/test/unit ]; then
echo Compiling $dir/test/unit/coffee;
$COFFEE -o $dir/test/unit/js -c $dir/test/unit/coffee;
fi
done

View File

@@ -0,0 +1,26 @@
#!/usr/bin/env python2
from os import listdir
from os.path import isfile, isdir, join
volumes = []
for module in listdir("modules/"):
if module[0] != '.':
if isfile(join("modules", module, 'index.coffee')):
volumes.append(join("modules", module, 'index.coffee'))
for directory in ['app/coffee', 'app/views', 'public/coffee', 'test/unit/coffee', 'test/acceptance/coffee', 'test/acceptance/config', 'test/acceptance/files']:
if isdir(join("modules", module, directory)):
volumes.append(join("modules", module, directory))
volumes_string = map(lambda vol: "- ./" + vol + ":/app/" + vol + ":ro", volumes)
volumes_string = "\n ".join(volumes_string)
with open("docker-shared.template.yml", "r") as f:
docker_shared_file = f.read()
docker_shared_file = docker_shared_file.replace("MODULE_VOLUMES", volumes_string)
with open("docker-shared.yml", "w") as f:
f.write(docker_shared_file)

14
services/web/bin/unit_test Executable file
View File

@@ -0,0 +1,14 @@
#!/bin/bash
set -e;
MOCHA="node_modules/.bin/mocha --recursive --reporter spec"
$MOCHA "$@" test/unit/js
for dir in modules/*;
do
if [ -d $dir/test/unit/js ]; then
$MOCHA "$@" $dir/test/unit/js
fi
done

View File

@@ -35,12 +35,12 @@ module.exports = settings =
# Databases
# ---------
mongo:
url : 'mongodb://127.0.0.1/sharelatex'
url : process.env['MONGO_URL'] || "mongodb://127.0.0.1/sharelatex"
redis:
web:
host: "localhost"
port: "6379"
host: process.env['REDIS_HOST'] || "localhost"
port: process.env['REDIS_PORT'] || "6379"
password: ""
# websessions:
@@ -74,8 +74,8 @@ module.exports = settings =
# ]
api:
host: "localhost"
port: "6379"
host: process.env['REDIS_HOST'] || "localhost"
port: process.env['REDIS_PORT'] || "6379"
password: ""
# Service locations
@@ -87,6 +87,7 @@ module.exports = settings =
internal:
web:
port: webPort = 3000
host: process.env['LISTEN_ADDRESS'] or 'localhost'
documentupdater:
port: docUpdaterPort = 3003
@@ -99,7 +100,7 @@ module.exports = settings =
user: httpAuthUser
pass: httpAuthPass
documentupdater:
url : "http://localhost:#{docUpdaterPort}"
url : "http://#{process.env['DOCUPDATER_HOST'] or 'localhost'}:#{docUpdaterPort}"
thirdPartyDataStore:
url : "http://localhost:3002"
emptyProjectFlushDelayMiliseconds: 5 * seconds
@@ -113,7 +114,7 @@ module.exports = settings =
enabled: false
url : "http://localhost:3054"
docstore:
url : "http://localhost:3016"
url : "http://#{process.env['DOCSTORE_HOST'] or 'localhost'}:3016"
pubUrl: "http://localhost:3016"
chat:
url: "http://localhost:3010"

View File

@@ -0,0 +1,36 @@
version: "2"
volumes:
node_modules:
services:
npm:
extends:
file: docker-shared.yml
service: app
command: npm install
test_unit:
extends:
file: docker-shared.yml
service: app
command: npm run test:unit
test_acceptance:
extends:
file: docker-shared.yml
service: app
environment:
REDIS_HOST: redis
MONGO_URL: "mongodb://mongo/sharelatex"
SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true'
depends_on:
- redis
- mongo
command: npm run start
redis:
image: redis
mongo:
image: mongo:3.4.6

View File

@@ -0,0 +1,30 @@
version: "2"
# We mount all the directories explicitly so that we are only mounting
# the coffee directories, so that the compiled js is only written inside
# the container, and not back to the local filesystem, where it would be
# root owned, and conflict with working outside of the container.
services:
app:
image: node:6.9.5
volumes:
- ./package.json:/app/package.json
- ./npm-shrinkwrap.json:/app/npm-shrinkwrap.json
- node_modules:/app/node_modules
- ./bin:/app/bin
# Copying the whole public dir is fine for now, and needed for
# some unit tests to pass, but we will want to isolate the coffee
# and vendor js files, so that the compiled js files are not written
# back to the local filesystem.
- ./public:/app/public
- ./app.coffee:/app/app.coffee:ro
- ./app/coffee:/app/app/coffee:ro
- ./app/templates:/app/app/templates:ro
- ./app/views:/app/app/views:ro
- ./config:/app/config
- ./test/unit/coffee:/app/test/unit/coffee:ro
- ./test/acceptance/coffee:/app/test/acceptance/coffee:ro
- ./test/smoke/coffee:/app/test/smoke/coffee:ro
MODULE_VOLUMES
working_dir: /app

View File

@@ -1,12 +1,6 @@
*/app/js
*/test/unit/js
*/index.js
ldap
admin-panel
groovehq
launchpad
learn-wiki
references-search
sharelatex-saml
templates
tpr-webmodule
# Ignore all modules except for a whitelist
*
!dropbox
!github-sync
!public-registration
!.gitignore

View File

@@ -9,6 +9,17 @@
"directories": {
"public": "./public"
},
"scripts": {
"test:acceptance:wait_for_app": "echo 'Waiting for app to be accessible' && while (! curl -s -o /dev/null localhost:3000/status) do sleep 1; done",
"test:acceptance:run": "bin/acceptance_test $@",
"test:acceptance:dir": "npm -q run compile:acceptance_tests && npm -q run test:acceptance:wait_for_app && npm -q run test:acceptance:run -- $@",
"test:acceptance": "npm -q run test:acceptance:dir -- $@ test/acceptance/js",
"test:unit": "npm -q run compile:app && npm -q run compile:unit_tests && bin/unit_test $@",
"compile:unit_tests": "bin/compile_unit_tests",
"compile:acceptance_tests": "bin/compile_acceptance_tests",
"compile:app": "bin/compile_app",
"start": "npm -q run compile:app && node app.js"
},
"dependencies": {
"archiver": "0.9.0",
"async": "0.6.2",
@@ -98,6 +109,7 @@
"grunt-postcss": "^0.8.0",
"grunt-sed": "^0.1.1",
"grunt-shell": "^2.1.0",
"mkdirp": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"sandboxed-module": "0.2.0",
"sinon": "^1.17.0",
"timekeeper": "",

View File

@@ -1,6 +1,6 @@
define [
"base"
"libs/fineuploader"
"fineuploader"
], (App, qq) ->
App.directive 'fineUpload', ($timeout) ->
return {

View File

@@ -29,7 +29,7 @@ define () ->
packageSnippets.push {
caption: "\\usepackage{}"
snippet: "\\usepackage{}"
snippet: "\\usepackage{$1}"
meta: "pkg"
score: 70
}

View File

@@ -22,7 +22,7 @@ define [
end = change.end
range = new Range(end.row, 0, end.row, end.column)
lineUpToCursor = @editor.getSession().getTextRange range
if lineUpToCursor.trim() == '%' or lineUpToCursor.startsWith '\\'
if lineUpToCursor.trim() == '%' or lineUpToCursor.slice(0, 1) == '\\'
range = new Range(end.row, 0, end.row, end.column + 80)
lineUpToCursor = @editor.getSession().getTextRange range
commandFragment = getLastCommandFragment lineUpToCursor
@@ -44,9 +44,9 @@ define [
linesContainLabel or
linesContainReqPackage
lastCommandFragmentIsLabel = commandFragment?.startsWith '\\label{'
lastCommandFragmentIsPackage = commandFragment?.startsWith '\\usepackage'
lastCommandFragmentIsReqPack = commandFragment?.startsWith '\\RequirePackage'
lastCommandFragmentIsLabel = commandFragment?.slice(0, 7) == '\\label{'
lastCommandFragmentIsPackage = commandFragment?.slice(0, 11) == '\\usepackage'
lastCommandFragmentIsReqPack = commandFragment?.slice(0, 15) == '\\RequirePackage'
lastCommandFragmentIsMeta =
lastCommandFragmentIsPackage or
lastCommandFragmentIsLabel or

View File

@@ -1,12 +1,10 @@
define [
"moment"
"libs/angular-autocomplete/angular-autocomplete"
"libs/ui-bootstrap"
"libs/ng-context-menu-0.1.4"
"libs/underscore-1.3.3"
"libs/algolia-2.5.2"
"libs/jquery.storage"
"libs/fineuploader"
"libs/angular-sanitize-1.2.17"
"libs/angular-cookie"
"libs/passfield"

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,4 @@
Settings = require('settings-sharelatex')
redis = require('redis-sharelatex')
logger = require("logger-sharelatex")
Async = require('async')

View File

@@ -1,4 +1,4 @@
BASE_URL = "http://localhost:3000"
BASE_URL = "http://#{process.env["HTTP_TEST_HOST"] or "localhost"}:3000"
module.exports = require("request").defaults({
baseUrl: BASE_URL,
followRedirect: false