diff --git a/services/web/Dockerfile b/services/web/Dockerfile index e560a11896..6bb8c61625 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -5,11 +5,17 @@ WORKDIR /app # install_deps changes app files and installs npm packages # as such it has to run at a later stage -FROM base as app +RUN apt-get update \ +&& apt-get install -y parallel \ +&& rm -rf /var/lib/apt/lists/* + +FROM base as deps COPY package.json package-lock.json /app/ -RUN npm install --quiet +RUN npm ci --quiet + +FROM deps as app COPY . /app diff --git a/services/web/Makefile b/services/web/Makefile index 2f6e5e44b6..24b12d07c5 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -48,6 +48,13 @@ test_unit_app: COMPOSE_PROJECT_NAME=unit_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) run --name unit_test_$(BUILD_DIR_NAME) --rm test_unit COMPOSE_PROJECT_NAME=unit_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down -v -t 0 +test_unit_app_parallel: export COMPOSE_PROJECT_NAME = \ + unit_test_parallel_$(BUILD_DIR_NAME) +test_unit_app_parallel: + $(DOCKER_COMPOSE) down -v -t 0 + $(DOCKER_COMPOSE) run --rm test_unit npm run test:unit:app:parallel + $(DOCKER_COMPOSE) down -v -t 0 + TEST_UNIT_MODULES = $(MODULE_DIRS:=/test_unit) $(TEST_UNIT_MODULES): %/test_unit: %/Makefile test_unit_modules: $(TEST_UNIT_MODULES) @@ -102,23 +109,46 @@ ci: # # Lint & format # +ORG_PATH = /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +RUN_LINT_FORMAT ?= \ + docker run --rm \ + --volume $(PWD):/src \ + --workdir /src \ + --env NODE_PATH=/app/node_modules \ + --env PATH=$(ORG_PATH):/app/node_modules/.bin \ + gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-deps format: npm -q run format +format_in_docker: + $(RUN_LINT_FORMAT) make format + format_fix: npm -q run format:fix lint: npm -q run lint +lint_in_docker: + $(RUN_LINT_FORMAT) make lint + # # Build & publish # -build: +build_deps: + docker build --pull \ + --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-deps \ + --cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-deps \ + --cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):master-deps \ + --target deps \ + . + +build: build_deps docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ + --cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-deps \ --build-arg SENTRY_RELEASE=${COMMIT_SHA} \ --build-arg BRANCH_NAME=$(BRANCH_NAME) \ . diff --git a/services/web/Makefile.module b/services/web/Makefile.module index a0f649c5fe..5555e94e62 100644 --- a/services/web/Makefile.module +++ b/services/web/Makefile.module @@ -21,10 +21,16 @@ DOCKER_COMPOSE_TEST_UNIT := \ export COMPOSE_PROJECT_NAME=unit_test_$(BUILD_DIR_NAME)_$(MODULE_NAME) \ && $(DOCKER_COMPOSE) +ifeq (,$(wildcard test/unit)) +test_unit: + +else test_unit: ${DOCKER_COMPOSE_TEST_UNIT} run --rm test_unit npm -q run test:unit:run_dir -- ${MOCHA_ARGS} $(MODULE_DIR)/test/unit/src ${DOCKER_COMPOSE_TEST_UNIT} down +endif + ifeq (,$(wildcard test/acceptance)) test_acceptance: clean_test_acceptance: diff --git a/services/web/bin/cdn_upload b/services/web/bin/cdn_upload index c3e82a4217..a500186171 100755 --- a/services/web/bin/cdn_upload +++ b/services/web/bin/cdn_upload @@ -3,9 +3,9 @@ set -e # Upload to staging CDN if branch is either 'master' or 'staging-master' if [[ "$BRANCH_NAME" == "master" || "$BRANCH_NAME" == "staging-master" ]]; then - tar --directory=/tmp/ -xzf build.tar.gz ./public/ + tar --directory=/tmp/ -xf build.tar gsutil -h "Cache-Control:public, max-age=31536000" -m cp -r /tmp/public $CDN_STAG - # Only upload to production CDN if branch is + # Only upload to production CDN if branch is if [[ "$BRANCH_NAME" == "master" ]]; then gsutil -h "Cache-Control:public, max-age=31536000" -m cp -r /tmp/public $CDN_PROD fi diff --git a/services/web/docker-compose.ci.yml b/services/web/docker-compose.ci.yml index bdbdaa2731..247d7e91bb 100644 --- a/services/web/docker-compose.ci.yml +++ b/services/web/docker-compose.ci.yml @@ -48,7 +48,7 @@ services: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER volumes: - ./:/tmp/build/ - command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . + command: tar -cf /tmp/build/build.tar public/ user: root redis: diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml index 952dacd6cc..87d1203ad3 100644 --- a/services/web/docker-compose.yml +++ b/services/web/docker-compose.yml @@ -1,4 +1,4 @@ -version: "2" +version: "2.3" volumes: data: @@ -6,7 +6,9 @@ volumes: services: test_unit: - image: node:10.19.0 + build: + context: . + target: base volumes: - .:/app working_dir: /app diff --git a/services/web/install_deps.sh b/services/web/install_deps.sh index 3049a97c70..104f38568e 100755 --- a/services/web/install_deps.sh +++ b/services/web/install_deps.sh @@ -1,15 +1,11 @@ #!/bin/bash -make --no-print-directory format & FORMAT=$! -make --no-print-directory lint & LINT=$! npm install git+https://github.com/sharelatex/translations-sharelatex.git#master & TRANSLATIONS=$! npm run webpack:production & WEBPACK=$! -echo "Waiting for lint, format, translations and minify to finish" +echo "Waiting for translations and minify to finish" -wait $LINT && echo "Lint complete" || exit 1 -wait $FORMAT && echo "Format complete" || exit 1 wait $TRANSLATIONS && echo "Translations install complete" || exit 1 wait $WEBPACK && echo "Webpack complete" || exit 1 diff --git a/services/web/package.json b/services/web/package.json index 9e26700e30..3c2b5db2d3 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -16,6 +16,7 @@ "test:acceptance:app": "npm run test:acceptance:run_dir -- test/acceptance/src", "test:unit:run_dir": "mocha --recursive --reporter spec --timeout 25000 --exit --grep=$MOCHA_GREP --file test/unit/bootstrap.js", "test:unit:app": "npm run test:unit:run_dir -- test/unit/src", + "test:unit:app:parallel": "parallel --plain --keep-order --halt now,fail=1 npm run test:unit:run_dir -- {} ::: test/unit/src/*", "test:frontend": "karma start", "test:frontend:single": "karma start --single-run", "start": "node $NODE_APP_OPTIONS app.js", diff --git a/services/web/test/unit/bootstrap.js b/services/web/test/unit/bootstrap.js index 73dd4dc0b1..69a9eb0d33 100644 --- a/services/web/test/unit/bootstrap.js +++ b/services/web/test/unit/bootstrap.js @@ -1,6 +1,9 @@ const chai = require('chai') const sinon = require('sinon') +// add chai.should() +chai.should() + // Load sinon-chai assertions so expect(stubFn).to.have.been.calledWith('abc') // has a nicer failure messages chai.use(require('sinon-chai'))