Update to build 1.0.0 build scripts

This commit is contained in:
James Allen
2018-02-15 16:28:40 +00:00
parent 6d44176ab5
commit 5d1659457e
22 changed files with 211 additions and 213 deletions
+1 -1
View File
@@ -1 +1 @@
6.9.5
6.9.5
-137
View File
@@ -1,137 +0,0 @@
module.exports = (grunt) ->
grunt.initConfig
forever:
app:
options:
index: "app.js"
execute:
app:
src: "app.js"
bunyan:
strict: false
coffee:
app_dir:
expand: true,
flatten: false,
cwd: 'app/coffee',
src: ['**/*.coffee'],
dest: 'app/js/',
ext: '.js'
app:
src: 'app.coffee'
dest: 'app.js'
acceptance_tests:
expand: true,
flatten: false,
cwd: 'test/acceptance/coffee',
src: ['**/*.coffee'],
dest: 'test/acceptance/js/',
ext: '.js'
unit_tests:
expand: true,
flatten: false,
cwd: 'test/unit/coffee',
src: ['**/*.coffee'],
dest: 'test/unit/js/',
ext: '.js'
clean:
app: ["app/js"]
acceptance_tests: ["test/acceptance/js"]
unit_tests: ["test/unit/js"]
mochaTest:
unit:
src: ["test/unit/js/#{grunt.option('feature') or '**'}/*.js"]
options:
reporter: grunt.option('reporter') or 'spec'
grep: grunt.option("grep")
acceptance:
src: ["test/acceptance/js/#{grunt.option('feature') or '*'}.js"]
options:
reporter: grunt.option('reporter') or 'spec'
grep: grunt.option("grep")
timeout: 50000
shell:
fullAcceptanceTests:
command: "bash ./test/acceptance/scripts/full-test.sh"
dockerTests:
command: 'docker run -v "$(pwd):/app" --rm sl-acceptance-test-runner'
availabletasks:
tasks:
options:
filter: 'exclude',
tasks: [
'coffee'
'clean'
'mochaTest'
'availabletasks'
'execute'
'bunyan'
]
groups:
"Compile tasks": [
"compile:server"
"compile:tests"
"compile"
"compile:unit_tests"
"compile:acceptance_tests"
"install"
]
"Test tasks": [
"test:unit"
"test:acceptance"
]
"Run tasks": [
"run"
"default"
]
"Misc": [
"help"
]
grunt.loadNpmTasks 'grunt-contrib-coffee'
grunt.loadNpmTasks 'grunt-contrib-clean'
grunt.loadNpmTasks 'grunt-mocha-test'
grunt.loadNpmTasks 'grunt-available-tasks'
grunt.loadNpmTasks 'grunt-execute'
grunt.loadNpmTasks 'grunt-bunyan'
grunt.loadNpmTasks 'grunt-forever'
grunt.loadNpmTasks 'grunt-shell'
grunt.registerTask 'help', 'Display this help list', 'availabletasks'
grunt.registerTask 'compile:server', 'Compile the server side coffee script', ['clean:app', 'coffee:app', 'coffee:app_dir']
grunt.registerTask 'compile:unit_tests', 'Compile the unit tests', ['clean:unit_tests', 'coffee:unit_tests']
grunt.registerTask 'compile:acceptance_tests', 'Compile the acceptance tests', ['clean:acceptance_tests', 'coffee:acceptance_tests']
grunt.registerTask 'compile:tests', 'Compile all the tests', ['compile:acceptance_tests', 'compile:unit_tests']
grunt.registerTask 'compile', 'Compiles everything need to run document-updater-sharelatex', ['compile:server']
grunt.registerTask 'install', "Compile everything when installing as an npm module", ['compile']
grunt.registerTask 'test:unit', 'Run the unit tests (use --grep=<regex> for individual tests)', ['compile:server', 'compile:unit_tests', 'mochaTest:unit']
grunt.registerTask(
'test:acceptance:full',
"Start server and run acceptance tests",
['shell:fullAcceptanceTests']
)
grunt.registerTask(
'test:acceptance:docker',
"Run acceptance tests inside docker container",
['shell:dockerTests']
)
grunt.registerTask 'test:acceptance', 'Run the acceptance tests (use --grep=<regex> for individual tests)', ['compile:acceptance_tests', 'mochaTest:acceptance']
grunt.registerTask 'run', "Compile and run the document-updater-sharelatex server", ['compile', 'bunyan', 'execute']
grunt.registerTask 'default', 'run'
+31 -26
View File
@@ -1,10 +1,11 @@
pipeline {
String cron_string = BRANCH_NAME == "master" ? "@daily" : ""
pipeline {
agent any
triggers {
pollSCM('* * * * *')
cron('@daily')
cron(cron_string)
}
stages {
@@ -17,15 +18,17 @@ pipeline {
}
}
steps {
// we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id
// which does not exist in the container's /etc/passwd file, causing the clone to fail.
// we need to disable logallrefupdates, else git clones
// during the npm install will require git to lookup the
// user id which does not exist in the container's
// /etc/passwd file, causing the clone to fail.
sh 'git config --global core.logallrefupdates false'
sh 'rm -fr node_modules'
sh 'rm -rf node_modules'
sh 'npm install && npm rebuild'
sh 'npm install --quiet grunt-cli'
}
}
stage('Compile and Test') {
stage('Compile') {
agent {
docker {
image 'node:6.9.5'
@@ -33,47 +36,49 @@ pipeline {
}
}
steps {
sh 'node_modules/.bin/grunt compile'
sh 'node_modules/.bin/grunt compile:acceptance_tests'
sh 'NODE_ENV=development node_modules/.bin/grunt test:unit'
sh 'npm run compile:all'
}
}
stage('Unit Tests') {
steps {
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
}
}
stage('Acceptance Tests') {
steps {
sh 'docker pull sharelatex/acceptance-test-runner'
sh 'docker run --rm -e SHARELATEX_ENABLE_PROJECT_HISTORY=true -v $(pwd):/app sharelatex/acceptance-test-runner'
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
}
}
stage('Package') {
stage('Package and publish build') {
steps {
sh 'echo ${BUILD_NUMBER} > build_number.txt'
sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
}
}
stage('Publish') {
steps {
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
// The deployment process uses this file to figure out the latest build
s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
}
}
}
stage('Sync OSS') {
when {
branch 'master'
}
stage('Publish build number') {
steps {
sshagent (credentials: ['GIT_DEPLOY_KEY']) {
sh 'git push git@github.com:sharelatex/document-updater-sharelatex.git HEAD:master'
sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
// The deployment process uses this file to figure out the latest build
s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
}
}
}
}
post {
always {
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean'
}
failure {
mail(from: "${EMAIL_ALERT_FROM}",
to: "${EMAIL_ALERT_TO}",
+29
View File
@@ -0,0 +1,29 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.0.0
BUILD_NUMBER ?= local
BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
PROJECT_NAME = document-updater
DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
DOCKER_COMPOSE := docker-compose ${DOCKER_COMPOSE_FLAGS}
clean:
rm -f app.js
rm -rf app/js
rm -rf test/unit/js
rm -rf test/acceptance/js
test: test_unit test_acceptance
test_unit:
@[ -d test/unit ] && $(DOCKER_COMPOSE) run --rm test_unit -- ${MOCHA_ARGS} || echo "document-updater has no unit tests"
test_acceptance: test_clean # clear the database before each acceptance test run
@[ -d test/acceptance ] && $(DOCKER_COMPOSE) run --rm test_acceptance -- ${MOCHA_ARGS} || echo "document-updater has no acceptance tests"
test_clean:
$(DOCKER_COMPOSE) down
.PHONY: clean test test_unit test_acceptance test_clean build publish
+4 -2
View File
@@ -103,8 +103,10 @@ shutdownCleanly = (signal) ->
port = Settings.internal?.documentupdater?.port or Settings.apis?.documentupdater?.port or 3003
host = Settings.internal.documentupdater.host or "localhost"
app.listen port, host, ->
logger.info "Document-updater starting up, listening on #{host}:#{port}"
if !module.parent # Called directly
app.listen port, host, ->
logger.info "Document-updater starting up, listening on #{host}:#{port}"
module.exports = app
for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT']
process.on signal, shutdownCleanly(signal)
@@ -16,7 +16,7 @@ module.exports =
trackchanges:
url: "http://#{process.env["TRACK_CHANGES_HOST"] or "localhost"}:3015"
project_history:
enabled: process.env.SHARELATEX_ENABLE_PROJECT_HISTORY == 'true'
enabled: true
url: "http://#{process.env["PROJECT_HISTORY_HOST"] or "localhost"}:3054"
redis:
@@ -0,0 +1,33 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.0.0
version: "2"
services:
test_unit:
image: node:6.9.5
volumes:
- .:/app
working_dir: /app
entrypoint: npm run test:unit:_run
test_acceptance:
image: node:6.9.5
volumes:
- .:/app
working_dir: /app
environment:
REDIS_HOST: redis
MONGO_HOST: mongo
depends_on:
- redis
- mongo
entrypoint: npm run test:acceptance:_run
redis:
image: redis
mongo:
image: mongo:3.4
@@ -0,0 +1,33 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.0.0
version: "2"
services:
test_unit:
image: node:6.9.5
volumes:
- .:/app
working_dir: /app
entrypoint: npm run test:unit
test_acceptance:
image: node:6.9.5
volumes:
- .:/app
environment:
REDIS_HOST: redis
MONGO_HOST: mongo
depends_on:
- redis
- mongo
working_dir: /app
entrypoint: npm run test:acceptance
redis:
image: redis
mongo:
image: mongo:3.4
+15
View File
@@ -0,0 +1,15 @@
{
"ignore": [
".git",
"node_modules/"
],
"verbose": true,
"execMap": {
"js": "npm run start"
},
"watch": [
"app/coffee/",
"app.coffee"
],
"ext": "coffee"
}
+10 -10
View File
@@ -8,7 +8,15 @@
},
"scripts": {
"compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
"start": "npm run compile:app && node app.js"
"start": "npm run compile:app && node app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@",
"compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'",
"compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'",
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests",
"nodemon": "nodemon --config nodemon.json"
},
"dependencies": {
"async": "^2.5.0",
@@ -30,15 +38,7 @@
"chai": "^3.5.0",
"chai-spies": "^0.7.1",
"cluster-key-slot": "^1.0.5",
"grunt": "~0.4.2",
"grunt-available-tasks": "~0.4.1",
"grunt-bunyan": "~0.5.0",
"grunt-contrib-clean": "~0.5.0",
"grunt-contrib-coffee": "~0.10.0",
"grunt-execute": "~0.1.5",
"grunt-forever": "^0.4.7",
"grunt-mocha-test": "~0.9.0",
"grunt-shell": "^1.3.0",
"mocha": "^5.0.1",
"timekeeper": "^2.0.0"
}
}
@@ -13,9 +13,10 @@ ProjectHistoryKeys = Settings.redis.project_history.key_schema
MockTrackChangesApi = require "./helpers/MockTrackChangesApi"
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
DocUpdaterApp = require "./helpers/DocUpdaterApp"
describe "Applying updates to a doc", ->
before ->
before (done) ->
@lines = ["one", "two", "three"]
@version = 42
@update =
@@ -26,6 +27,7 @@ describe "Applying updates to a doc", ->
}]
v: @version
@result = ["one", "one and a half", "two", "three"]
DocUpdaterApp.ensureRunning(done)
describe "when the document is not loaded", ->
before (done) ->
@@ -7,6 +7,7 @@ ProjectHistoryKeys = Settings.redis.project_history.key_schema
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
DocUpdaterApp = require "./helpers/DocUpdaterApp"
describe "Applying updates to a project's structure", ->
before ->
@@ -20,9 +21,11 @@ describe "Applying updates to a project's structure", ->
pathname: '/file-path'
newPathname: '/new-file-path'
@fileUpdates = [ @fileUpdate ]
DocUpdaterClient.sendProjectUpdate @project_id, @user_id, [], @fileUpdates, (error) ->
DocUpdaterApp.ensureRunning (error) =>
throw error if error?
setTimeout done, 200
DocUpdaterClient.sendProjectUpdate @project_id, @user_id, [], @fileUpdates, (error) ->
throw error if error?
setTimeout done, 200
it "should push the applied file renames to the project history api", (done) ->
rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) =>
@@ -6,9 +6,10 @@ MockTrackChangesApi = require "./helpers/MockTrackChangesApi"
MockProjectHistoryApi = require "./helpers/MockProjectHistoryApi"
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
DocUpdaterApp = require "./helpers/DocUpdaterApp"
describe "Deleting a document", ->
before ->
before (done) ->
@lines = ["one", "two", "three"]
@version = 42
@update =
@@ -22,6 +23,7 @@ describe "Deleting a document", ->
sinon.spy MockTrackChangesApi, "flushDoc"
sinon.spy MockProjectHistoryApi, "flushProject"
DocUpdaterApp.ensureRunning(done)
after ->
MockTrackChangesApi.flushDoc.restore()
@@ -7,9 +7,10 @@ MockTrackChangesApi = require "./helpers/MockTrackChangesApi"
MockProjectHistoryApi = require "./helpers/MockProjectHistoryApi"
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
DocUpdaterApp = require "./helpers/DocUpdaterApp"
describe "Deleting a project", ->
before ->
before (done) ->
@project_id = DocUpdaterClient.randomId()
@docs = [{
id: doc_id0 = DocUpdaterClient.randomId()
@@ -42,6 +43,7 @@ describe "Deleting a project", ->
sinon.spy MockTrackChangesApi, "flushDoc"
sinon.spy MockProjectHistoryApi, "flushProject"
DocUpdaterApp.ensureRunning(done)
after ->
MockTrackChangesApi.flushDoc.restore()
@@ -5,9 +5,10 @@ async = require "async"
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
DocUpdaterApp = require "./helpers/DocUpdaterApp"
describe "Flushing a project", ->
before ->
before (done) ->
@project_id = DocUpdaterClient.randomId()
@docs = [{
id: doc_id0 = DocUpdaterClient.randomId()
@@ -37,6 +38,7 @@ describe "Flushing a project", ->
lines: doc.lines
version: doc.update.v
}
DocUpdaterApp.ensureRunning(done)
describe "with documents which have been updated", ->
before (done) ->
@@ -6,9 +6,10 @@ async = require "async"
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
DocUpdaterApp = require "./helpers/DocUpdaterApp"
describe "Flushing a doc to Mongo", ->
before ->
before (done) ->
@lines = ["one", "two", "three"]
@version = 42
@update =
@@ -19,6 +20,7 @@ describe "Flushing a doc to Mongo", ->
}]
v: @version
@result = ["one", "one and a half", "two", "three"]
DocUpdaterApp.ensureRunning(done)
describe "when the updated doc exists in the doc updater", ->
before (done) ->
@@ -5,12 +5,13 @@ expect = chai.expect
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
DocUpdaterApp = require "./helpers/DocUpdaterApp"
describe "Getting a document", ->
before (done) ->
@lines = ["one", "two", "three"]
@version = 42
setTimeout done, 200 # Give MockWebApi a chance to start
DocUpdaterApp.ensureRunning(done)
describe "when the document is not loaded", ->
before (done) ->
@@ -5,12 +5,13 @@ expect = chai.expect
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
DocUpdaterApp = require "./helpers/DocUpdaterApp"
describe "Getting documents for project", ->
before (done) ->
@lines = ["one", "two", "three"]
@version = 42
setTimeout done, 200 # Give MockWebApi a chance to start
DocUpdaterApp.ensureRunning(done)
describe "when project state hash does not match", ->
before (done) ->
@@ -6,6 +6,7 @@ async = require "async"
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
DocUpdaterApp = require "./helpers/DocUpdaterApp"
describe "Ranges", ->
describe "tracking changes from ops", ->
@@ -41,11 +42,14 @@ describe "Ranges", ->
for update in @updates
do (update) =>
jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback
DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) =>
DocUpdaterApp.ensureRunning (error) =>
throw error if error?
async.series jobs, (error) ->
DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) =>
throw error if error?
setTimeout done, 200
async.series jobs, (error) ->
throw error if error?
done()
it "should update the ranges", (done) ->
DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) =>
@@ -10,9 +10,10 @@ MockTrackChangesApi = require "./helpers/MockTrackChangesApi"
MockProjectHistoryApi = require "./helpers/MockProjectHistoryApi"
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
DocUpdaterApp = require "./helpers/DocUpdaterApp"
describe "Setting a document", ->
before ->
before (done) ->
@lines = ["one", "two", "three"]
@version = 42
@update =
@@ -30,6 +31,7 @@ describe "Setting a document", ->
sinon.spy MockTrackChangesApi, "flushDoc"
sinon.spy MockProjectHistoryApi, "flushProject"
sinon.spy MockWebApi, "setDocument"
DocUpdaterApp.ensureRunning(done)
after ->
MockTrackChangesApi.flushDoc.restore()
@@ -0,0 +1,20 @@
app = require('../../../../app')
require("logger-sharelatex").logger.level("fatal")
module.exports =
running: false
initing: false
callbacks: []
ensureRunning: (callback = (error) ->) ->
if @running
return callback()
else if @initing
@callbacks.push callback
else
@initing = true
@callbacks.push callback
app.listen 3003, "localhost", (error) =>
throw error if error?
@running = true
for callback in @callbacks
callback()
@@ -1,23 +0,0 @@
#! /usr/bin/env bash
# npm rebuild
echo ">> Starting server..."
grunt --no-color forever:app:start
echo ">> Server started"
sleep 5
echo ">> Running acceptance tests..."
grunt --no-color mochaTest:acceptance
_test_exit_code=$?
echo ">> Killing server"
grunt --no-color forever:app:stop
echo ">> Done"
exit $_test_exit_code