diff --git a/services/docstore/.dockerignore b/services/docstore/.dockerignore new file mode 100644 index 0000000000..386f26df30 --- /dev/null +++ b/services/docstore/.dockerignore @@ -0,0 +1,9 @@ +node_modules/* +gitrev +.git +.gitignore +.npm +.nvmrc +nodemon.json +app.js +**/js/* diff --git a/services/docstore/Dockerfile b/services/docstore/Dockerfile new file mode 100644 index 0000000000..9ead057334 --- /dev/null +++ b/services/docstore/Dockerfile @@ -0,0 +1,22 @@ +FROM node:6.14.1 as app + +WORKDIR /app + +#wildcard as some files may not be in all repos +COPY package*.json npm-shrink*.json /app/ + +RUN npm install --quiet + +COPY . /app + + +RUN npm run compile:all + +FROM node:6.14.1 + +COPY --from=app /app /app + +WORKDIR /app +USER node + +CMD ["node","app.js"] diff --git a/services/docstore/Jenkinsfile b/services/docstore/Jenkinsfile index 5eda26c7a3..bc9ba0142f 100644 --- a/services/docstore/Jenkinsfile +++ b/services/docstore/Jenkinsfile @@ -1,62 +1,42 @@ -pipeline { +String cron_string = BRANCH_NAME == "master" ? "@daily" : "" +pipeline { agent any - + triggers { pollSCM('* * * * *') - cron('@daily') + cron(cron_string) } stages { - stage('Install') { - agent { - docker { - image 'node:6.14.1' - args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" - reuseNode true - } - } + stage('Build') { steps { - // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id - // which does not exist in the container's /etc/passwd file, causing the clone to fail. - sh 'git config --global core.logallrefupdates false' - sh 'rm -fr node_modules' - sh 'npm install && npm rebuild' - sh 'npm install --quiet grunt-cli' + sh 'make build' } } - stage('Compile and Test') { - agent { - docker { - image 'node:6.14.1' - reuseNode true - } - } + + stage('Unit Tests') { steps { - sh 'node_modules/.bin/grunt install' - sh 'node_modules/.bin/grunt compile:acceptance_tests' - sh 'NODE_ENV=development node_modules/.bin/grunt test:unit' + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' } } + stage('Acceptance Tests') { steps { - sh 'docker pull sharelatex/acceptance-test-runner' - withCredentials([usernamePassword(credentialsId: 'S3_DOCSTORE_TEST_AWS_KEYS', passwordVariable: 'AWS_SECRET', usernameVariable: 'AWS_ID')]) { - sh 'docker run --rm -e AWS_BUCKET="sl-doc-archive-testing" -e AWS_ACCESS_KEY_ID=$AWS_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET -v $(pwd):/app sharelatex/acceptance-test-runner' - } + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance' } } - stage('Package') { + + stage('Package and publish build') { steps { - sh 'echo ${BUILD_NUMBER} > build_number.txt' - sh 'touch build.tar.gz' // Avoid tar warning about files changing during read - sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .' + sh 'make publish' } } - stage('Publish') { + + stage('Publish build number') { steps { + sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt' withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { - s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") // The deployment process uses this file to figure out the latest build s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") } @@ -65,6 +45,10 @@ pipeline { } post { + always { + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean' + } + failure { mail(from: "${EMAIL_ALERT_FROM}", to: "${EMAIL_ALERT_TO}", diff --git a/services/docstore/Makefile b/services/docstore/Makefile new file mode 100644 index 0000000000..b3ba01358c --- /dev/null +++ b/services/docstore/Makefile @@ -0,0 +1,42 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +BUILD_NUMBER ?= local +BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) +PROJECT_NAME = docstore +DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml +DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ + BRANCH_NAME=$(BRANCH_NAME) \ + PROJECT_NAME=$(PROJECT_NAME) \ + MOCHA_GREP=${MOCHA_GREP} \ + docker-compose ${DOCKER_COMPOSE_FLAGS} + + +clean: + rm -f app.js + rm -rf app/js + rm -rf test/unit/js + rm -rf test/acceptance/js + +test: test_unit test_acceptance + +test_unit: + @[ ! -d test/unit ] && echo "docstore has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit + +test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run + @[ ! -d test/acceptance ] && echo "docstore has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance + +test_clean: + $(DOCKER_COMPOSE) down -v -t 0 + +test_acceptance_pre_run: + @[ ! -f test/acceptance/scripts/pre-run ] && echo "docstore has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run +build: + docker build --pull --tag gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) . + +publish: + docker push gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + +.PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/docstore/app.coffee b/services/docstore/app.coffee index f81b4f3ea8..eb74016ff3 100644 --- a/services/docstore/app.coffee +++ b/services/docstore/app.coffee @@ -52,6 +52,10 @@ app.use (error, req, res, next) -> port = Settings.internal.docstore.port host = Settings.internal.docstore.host -app.listen port, host, (error) -> - throw error if error? - logger.info "Docstore starting up, listening on #{host}:#{port}" + +if !module.parent # Called directly + app.listen port, host, (error) -> + throw error if error? + logger.info "Docstore starting up, listening on #{host}:#{port}" + +module.exports = app \ No newline at end of file diff --git a/services/docstore/docker-compose.ci.yml b/services/docstore/docker-compose.ci.yml new file mode 100644 index 0000000000..21c006641e --- /dev/null +++ b/services/docstore/docker-compose.ci.yml @@ -0,0 +1,32 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +version: "2" + +services: + test_unit: + image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + user: node + command: npm run test:unit:_run + + test_acceptance: + build: . + image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + POSTGRES_HOST: postgres + depends_on: + - mongo + - redis + user: node + command: npm run test:acceptance:_run + + redis: + image: redis + + mongo: + image: mongo:3.4 + diff --git a/services/docstore/docker-compose.yml b/services/docstore/docker-compose.yml new file mode 100644 index 0000000000..f24caa8883 --- /dev/null +++ b/services/docstore/docker-compose.yml @@ -0,0 +1,39 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +version: "2" + +services: + test_unit: + build: . + volumes: + - .:/app + working_dir: /app + environment: + MOCHA_GREP: ${MOCHA_GREP} + command: npm run test:unit + user: node + + test_acceptance: + build: . + volumes: + - .:/app + working_dir: /app + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + POSTGRES_HOST: postgres + MOCHA_GREP: ${MOCHA_GREP} + user: node + depends_on: + - mongo + - redis + command: npm run test:acceptance + redis: + image: redis + + mongo: + image: mongo:3.4 + diff --git a/services/docstore/nodemon.json b/services/docstore/nodemon.json new file mode 100644 index 0000000000..98db38d71b --- /dev/null +++ b/services/docstore/nodemon.json @@ -0,0 +1,19 @@ +{ + "ignore": [ + ".git", + "node_modules/" + ], + "verbose": true, + "legacyWatch": true, + "execMap": { + "js": "npm run start" + }, + + "watch": [ + "app/coffee/", + "app.coffee", + "config/" + ], + "ext": "coffee" + +} diff --git a/services/docstore/package.json b/services/docstore/package.json index ab64a7d3a3..2e9db708a1 100644 --- a/services/docstore/package.json +++ b/services/docstore/package.json @@ -8,8 +8,16 @@ "url": "https://github.com/sharelatex/docstore-sharelatex.git" }, "scripts": { - "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee", - "start": "npm run compile:app && node app.js" + "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", + "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", + "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", + "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", + "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", + "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests", + "nodemon": "nodemon --config nodemon.json" }, "dependencies": { "async": "~0.8.0", @@ -33,6 +41,7 @@ "grunt-execute": "~0.2.1", "grunt-forever": "~0.4.4", "grunt-mocha-test": "~0.10.2", + "mocha": "^4.0.1", "grunt-shell": "~0.7.0", "request": "~2.34.0", "sandboxed-module": "~0.3.0", diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee index 2c457922c0..0291eef032 100644 --- a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -6,11 +6,15 @@ async = require "async" Settings = require("settings-sharelatex") DocArchiveManager = require("../../../app/js/DocArchiveManager.js") request = require "request" - +DocstoreApp = require "./helpers/DocstoreApp" DocstoreClient = require "./helpers/DocstoreClient" describe "Archiving", -> + + before (done)-> + DocstoreApp.ensureRunning(done) + describe "multiple docs in a project", -> before (done) -> @project_id = ObjectId() @@ -29,6 +33,7 @@ describe "Archiving", -> do (doc) => (callback) => DocstoreClient.createDoc @project_id, doc._id, doc.lines, doc.version, doc.ranges, callback + async.series jobs, (error) => throw error if error? DocstoreClient.archiveAllDoc @project_id, (error, @res) => diff --git a/services/docstore/test/acceptance/coffee/DeletingDocsTests.coffee b/services/docstore/test/acceptance/coffee/DeletingDocsTests.coffee index 82dbfb4c66..f97a352c14 100644 --- a/services/docstore/test/acceptance/coffee/DeletingDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/DeletingDocsTests.coffee @@ -2,6 +2,7 @@ sinon = require "sinon" chai = require("chai") chai.should() {db, ObjectId} = require "../../../app/js/mongojs" +DocstoreApp = require "./helpers/DocstoreApp" DocstoreClient = require "./helpers/DocstoreClient" @@ -12,9 +13,10 @@ describe "Deleting a doc", -> @lines = ["original", "lines"] @version = 42 @ranges = [] - DocstoreClient.createDoc @project_id, @doc_id, @lines, @version, @ranges, (error) => - throw error if error? - done() + DocstoreApp.ensureRunning => + DocstoreClient.createDoc @project_id, @doc_id, @lines, @version, @ranges, (error) => + throw error if error? + done() describe "when the doc exists", -> beforeEach (done) -> diff --git a/services/docstore/test/acceptance/coffee/GettingAllDocsTests.coffee b/services/docstore/test/acceptance/coffee/GettingAllDocsTests.coffee index 7244693888..6a16006044 100644 --- a/services/docstore/test/acceptance/coffee/GettingAllDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/GettingAllDocsTests.coffee @@ -3,6 +3,7 @@ chai = require("chai") chai.should() {ObjectId} = require "mongojs" async = require "async" +DocstoreApp = require "./helpers/DocstoreApp" DocstoreClient = require "./helpers/DocstoreClient" @@ -39,6 +40,8 @@ describe "Getting all docs", -> jobs.push (cb) => DocstoreClient.createDoc @project_id, @deleted_doc._id, @deleted_doc.lines, version, @deleted_doc.ranges, (err)=> DocstoreClient.deleteDoc @project_id, @deleted_doc._id, cb + jobs.unshift (cb)-> + DocstoreApp.ensureRunning cb async.series jobs, done it "getAllDocs should return all the (non-deleted) docs", (done) -> diff --git a/services/docstore/test/acceptance/coffee/GettingDocsTests.coffee b/services/docstore/test/acceptance/coffee/GettingDocsTests.coffee index b17a294f24..dbdbd31aa2 100644 --- a/services/docstore/test/acceptance/coffee/GettingDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/GettingDocsTests.coffee @@ -2,6 +2,7 @@ sinon = require "sinon" chai = require("chai") chai.should() {ObjectId} = require "mongojs" +DocstoreApp = require "./helpers/DocstoreApp" DocstoreClient = require "./helpers/DocstoreClient" @@ -20,9 +21,10 @@ describe "Getting a doc", -> ts: new Date().toString() }] } - DocstoreClient.createDoc @project_id, @doc_id, @lines, @version, @ranges, (error) => - throw error if error? - done() + DocstoreApp.ensureRunning => + DocstoreClient.createDoc @project_id, @doc_id, @lines, @version, @ranges, (error) => + throw error if error? + done() describe "when the doc exists", -> it "should get the doc lines and version", (done) -> diff --git a/services/docstore/test/acceptance/coffee/UpdatingDocsTests.coffee b/services/docstore/test/acceptance/coffee/UpdatingDocsTests.coffee index f34a4e3e76..4d567e8f4c 100644 --- a/services/docstore/test/acceptance/coffee/UpdatingDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/UpdatingDocsTests.coffee @@ -2,6 +2,7 @@ sinon = require "sinon" chai = require("chai") chai.should() {ObjectId} = require "mongojs" +DocstoreApp = require "./helpers/DocstoreApp" DocstoreClient = require "./helpers/DocstoreClient" @@ -30,9 +31,10 @@ describe "Applying updates to a doc", -> }] } @version = 42 - DocstoreClient.createDoc @project_id, @doc_id, @originalLines, @version, @originalRanges, (error) => - throw error if error? - done() + DocstoreApp.ensureRunning => + DocstoreClient.createDoc @project_id, @doc_id, @originalLines, @version, @originalRanges, (error) => + throw error if error? + done() describe "when nothing has been updated", -> beforeEach (done) -> diff --git a/services/docstore/test/acceptance/coffee/helpers/DocstoreApp.coffee b/services/docstore/test/acceptance/coffee/helpers/DocstoreApp.coffee new file mode 100644 index 0000000000..4a916955a9 --- /dev/null +++ b/services/docstore/test/acceptance/coffee/helpers/DocstoreApp.coffee @@ -0,0 +1,21 @@ +app = require('../../../../app') +require("logger-sharelatex").logger.level("error") +settings = require("settings-sharelatex") + +module.exports = + running: false + initing: false + callbacks: [] + ensureRunning: (callback = (error) ->) -> + if @running + return callback() + else if @initing + @callbacks.push callback + else + @initing = true + @callbacks.push callback + app.listen settings.internal.docstore.port, "localhost", (error) => + throw error if error? + @running = true + for callback in @callbacks + callback() \ No newline at end of file