diff --git a/services/track-changes/.eslintrc b/services/track-changes/.eslintrc index 2e945d6ffb..76dad1561d 100644 --- a/services/track-changes/.eslintrc +++ b/services/track-changes/.eslintrc @@ -8,7 +8,7 @@ "prettier/standard" ], "parserOptions": { - "ecmaVersion": 2017 + "ecmaVersion": 2018 }, "plugins": [ "mocha", diff --git a/services/track-changes/.github/dependabot.yml b/services/track-changes/.github/dependabot.yml new file mode 100644 index 0000000000..c6f98d843d --- /dev/null +++ b/services/track-changes/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 +updates: + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "daily" + + pull-request-branch-name: + # Separate sections of the branch name with a hyphen + # Docker images use the branch name and do not support slashes in tags + # https://github.com/overleaf/google-ops/issues/822 + # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#pull-request-branch-nameseparator + separator: "-" + + # Block informal upgrades -- security upgrades use a separate queue. + # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#open-pull-requests-limit + open-pull-requests-limit: 0 diff --git a/services/track-changes/.gitignore b/services/track-changes/.gitignore index 72cd163116..008dc714ba 100644 --- a/services/track-changes/.gitignore +++ b/services/track-changes/.gitignore @@ -2,3 +2,6 @@ node_modules/ forever/ *.js.map + +# managed by dev-environment$ bin/update_build_scripts +.npmrc diff --git a/services/track-changes/Dockerfile b/services/track-changes/Dockerfile index 7b47fb9ee2..78a715757d 100644 --- a/services/track-changes/Dockerfile +++ b/services/track-changes/Dockerfile @@ -11,12 +11,10 @@ FROM base as app #wildcard as some files may not be in all repos COPY package*.json npm-shrink*.json /app/ -RUN npm install --quiet +RUN npm ci --quiet COPY . /app - - FROM base COPY --from=app /app /app diff --git a/services/track-changes/Jenkinsfile b/services/track-changes/Jenkinsfile deleted file mode 100644 index 71f95c3c14..0000000000 --- a/services/track-changes/Jenkinsfile +++ /dev/null @@ -1,131 +0,0 @@ -String cron_string = BRANCH_NAME == "master" ? "@daily" : "" - -pipeline { - agent any - - environment { - GIT_PROJECT = "track-changes" - JENKINS_WORKFLOW = "track-changes-sharelatex" - TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline" - GIT_API_URL = "https://api.github.com/repos/overleaf/${GIT_PROJECT}/statuses/$GIT_COMMIT" - } - - triggers { - pollSCM('* * * * *') - cron(cron_string) - } - - stages { - - stage('Install') { - steps { - withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { - sh "curl $GIT_API_URL \ - --data '{ \ - \"state\" : \"pending\", \ - \"target_url\": \"$TARGET_URL\", \ - \"description\": \"Your build is underway\", \ - \"context\": \"ci/jenkins\" }' \ - -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD" - } - } - } - - stage('Build') { - steps { - sh 'make build' - } - } - - stage('Linting') { - steps { - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format' - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint' - } - } - - stage('Unit Tests') { - steps { - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' - } - } - - stage('Acceptance Tests') { - steps { - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance' - } - } - - stage('Package and docker push') { - steps { - sh 'echo ${BUILD_NUMBER} > build_number.txt' - sh 'touch build.tar.gz' // Avoid tar warning about files changing during read - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make tar' - - withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) { - sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}' - } - sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish' - sh 'docker logout https://gcr.io/overleaf-ops' - - } - } - - stage('Publish to s3') { - steps { - sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt' - withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { - s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") - } - withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { - // The deployment process uses this file to figure out the latest build - s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") - } - } - } - } - - post { - always { - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean' - sh 'make clean' - } - - success { - withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { - sh "curl $GIT_API_URL \ - --data '{ \ - \"state\" : \"success\", \ - \"target_url\": \"$TARGET_URL\", \ - \"description\": \"Your build succeeded!\", \ - \"context\": \"ci/jenkins\" }' \ - -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD" - } - } - - failure { - mail(from: "${EMAIL_ALERT_FROM}", - to: "${EMAIL_ALERT_TO}", - subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}", - body: "Build: ${BUILD_URL}") - withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { - sh "curl $GIT_API_URL \ - --data '{ \ - \"state\" : \"failure\", \ - \"target_url\": \"$TARGET_URL\", \ - \"description\": \"Your build failed\", \ - \"context\": \"ci/jenkins\" }' \ - -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD" - } - } - } - - // The options directive is for configuration that applies to the whole job. - options { - // we'd like to make sure remove old builds, so we don't fill up our storage! - buildDiscarder(logRotator(numToKeepStr:'50')) - - // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after: - timeout(time: 30, unit: 'MINUTES') - } -} diff --git a/services/track-changes/Makefile b/services/track-changes/Makefile index e1b4843fea..eaff3428ca 100644 --- a/services/track-changes/Makefile +++ b/services/track-changes/Makefile @@ -5,6 +5,8 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = track-changes +BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') + DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ BRANCH_NAME=$(BRANCH_NAME) \ @@ -12,39 +14,63 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker-compose ${DOCKER_COMPOSE_FLAGS} +DOCKER_COMPOSE_TEST_ACCEPTANCE = \ + COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) + +DOCKER_COMPOSE_TEST_UNIT = \ + COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) + clean: docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) format: - $(DOCKER_COMPOSE) run --rm test_unit npm run format + $(DOCKER_COMPOSE) run --rm test_unit npm run --silent format format_fix: - $(DOCKER_COMPOSE) run --rm test_unit npm run format:fix + $(DOCKER_COMPOSE) run --rm test_unit npm run --silent format:fix lint: - $(DOCKER_COMPOSE) run --rm test_unit npm run lint + $(DOCKER_COMPOSE) run --rm test_unit npm run --silent lint test: format lint test_unit test_acceptance test_unit: - @[ ! -d test/unit ] && echo "track-changes has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit +ifneq (,$(wildcard test/unit)) + $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(MAKE) test_unit_clean +endif -test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run +test_clean: test_unit_clean +test_unit_clean: +ifneq (,$(wildcard test/unit)) + $(DOCKER_COMPOSE_TEST_UNIT) down -v -t 0 +endif -test_acceptance_debug: test_clean test_acceptance_pre_run test_acceptance_run_debug +test_acceptance: test_acceptance_clean test_acceptance_pre_run test_acceptance_run + $(MAKE) test_acceptance_clean + +test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_acceptance_run_debug + $(MAKE) test_acceptance_clean test_acceptance_run: - @[ ! -d test/acceptance ] && echo "track-changes has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance +ifneq (,$(wildcard test/acceptance)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance +endif test_acceptance_run_debug: - @[ ! -d test/acceptance ] && echo "track-changes has no acceptance tests" || $(DOCKER_COMPOSE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk +ifneq (,$(wildcard test/acceptance)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk +endif -test_clean: - $(DOCKER_COMPOSE) down -v -t 0 +test_clean: test_acceptance_clean +test_acceptance_clean: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: - @[ ! -f test/acceptance/js/scripts/pre-run ] && echo "track-changes has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run +ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run +endif build: docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ diff --git a/services/track-changes/buildscript.txt b/services/track-changes/buildscript.txt index 1551e8fbf3..1750deaa0e 100644 --- a/services/track-changes/buildscript.txt +++ b/services/track-changes/buildscript.txt @@ -1,10 +1,8 @@ track-changes ---acceptance-creds=None --dependencies=mongo,redis,s3 --docker-repos=gcr.io/overleaf-ops --env-add=AWS_BUCKET=bucket --env-pass-through= ---language=es --node-version=10.21.0 --public-repo=True ---script-version=2.0.0 +--script-version=3.3.2 diff --git a/services/track-changes/docker-compose.ci.yml b/services/track-changes/docker-compose.ci.yml index 4e41056770..6717530161 100644 --- a/services/track-changes/docker-compose.ci.yml +++ b/services/track-changes/docker-compose.ci.yml @@ -11,6 +11,7 @@ services: command: npm run test:unit:_run environment: NODE_ENV: test + NODE_OPTIONS: "--unhandled-rejections=strict" test_acceptance: @@ -27,6 +28,7 @@ services: AWS_SECRET_ACCESS_KEY: fake MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test + NODE_OPTIONS: "--unhandled-rejections=strict" AWS_BUCKET: bucket depends_on: mongo: @@ -50,10 +52,10 @@ services: image: redis mongo: - image: mongo:3.6 + image: mongo:4.0 s3: - image: adobe/s3mock + build: + context: test/acceptance/deps + dockerfile: Dockerfile.s3mock environment: - initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9090"] diff --git a/services/track-changes/docker-compose.yml b/services/track-changes/docker-compose.yml index 27652dedab..ba2b13aad6 100644 --- a/services/track-changes/docker-compose.yml +++ b/services/track-changes/docker-compose.yml @@ -13,7 +13,8 @@ services: environment: MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test - command: npm run test:unit + NODE_OPTIONS: "--unhandled-rejections=strict" + command: npm run --silent test:unit user: node test_acceptance: @@ -33,6 +34,7 @@ services: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ERROR NODE_ENV: test + NODE_OPTIONS: "--unhandled-rejections=strict" AWS_BUCKET: bucket user: node depends_on: @@ -42,17 +44,17 @@ services: condition: service_healthy s3: condition: service_healthy - command: npm run test:acceptance + command: npm run --silent test:acceptance redis: image: redis mongo: - image: mongo:3.6 + image: mongo:4.0 s3: - image: adobe/s3mock + build: + context: test/acceptance/deps + dockerfile: Dockerfile.s3mock environment: - initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9090"] diff --git a/services/track-changes/nodemon.json b/services/track-changes/nodemon.json index 5826281b84..e3e8817d90 100644 --- a/services/track-changes/nodemon.json +++ b/services/track-changes/nodemon.json @@ -8,7 +8,6 @@ "execMap": { "js": "npm run start" }, - "watch": [ "app/js/", "app.js", diff --git a/services/track-changes/package.json b/services/track-changes/package.json index 91e8c160b1..28f8d26406 100644 --- a/services/track-changes/package.json +++ b/services/track-changes/package.json @@ -13,7 +13,7 @@ "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "nodemon --config nodemon.json", - "lint": "node_modules/.bin/eslint .", + "lint": "node_modules/.bin/eslint --max-warnings 0 .", "format": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --list-different", "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write" }, @@ -45,8 +45,8 @@ "chai": "~4.2.0", "cli": "^1.0.1", "eslint": "^6.8.0", - "eslint-config-prettier": "^6.10.1", - "eslint-config-standard": "^14.1.1", + "eslint-config-prettier": "^6.10.0", + "eslint-config-standard": "^14.1.0", "eslint-config-standard-jsx": "^8.1.0", "eslint-config-standard-react": "^9.2.0", "eslint-plugin-chai-expect": "^2.1.0", @@ -61,7 +61,7 @@ "eslint-plugin-standard": "^4.0.1", "memorystream": "0.3.1", "mocha": "^7.1.1", - "prettier": "^2.0.1", + "prettier": "^2.0.0", "prettier-eslint-cli": "^5.0.0", "sandboxed-module": "~2.0.3", "sinon": "~9.0.1", diff --git a/services/track-changes/test/acceptance/deps/Dockerfile.s3mock b/services/track-changes/test/acceptance/deps/Dockerfile.s3mock new file mode 100644 index 0000000000..15eda4dd4b --- /dev/null +++ b/services/track-changes/test/acceptance/deps/Dockerfile.s3mock @@ -0,0 +1,4 @@ +FROM adobe/s3mock +RUN apk add --update --no-cache curl +COPY healthcheck.sh /healthcheck.sh +HEALTHCHECK --interval=1s --timeout=1s --retries=30 CMD /healthcheck.sh http://localhost:9090 diff --git a/services/track-changes/test/acceptance/deps/healthcheck.sh b/services/track-changes/test/acceptance/deps/healthcheck.sh new file mode 100644 index 0000000000..cd19cea637 --- /dev/null +++ b/services/track-changes/test/acceptance/deps/healthcheck.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +# health check to allow 404 status code as valid +STATUSCODE=$(curl --silent --output /dev/null --write-out "%{http_code}" $1) +# will be 000 on non-http error (e.g. connection failure) +if test $STATUSCODE -ge 500 || test $STATUSCODE -lt 200; then + exit 1 +fi +exit 0