diff --git a/dockerfiles/cypress/Dockerfile b/dockerfiles/cypress/Dockerfile index 98d478ede0..b6042a0ea3 100644 --- a/dockerfiles/cypress/Dockerfile +++ b/dockerfiles/cypress/Dockerfile @@ -2,10 +2,15 @@ FROM cypress/included:15.12.0 ARG USER_UID=1000 ARG USER_GID=1000 +# Corepack setup, shared between all the images. +ENV PATH="/overleaf/node_modules/.bin:$PATH" +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + WORKDIR /overleaf -RUN corepack enable \ -&& sed -i s/node:x:1000:/node:x:${USER_GID}:/ /etc/group \ +RUN sed -i s/node:x:1000:/node:x:${USER_GID}:/ /etc/group \ && sed -i s_node:x:1000:1000::/home/node:/bin/bash_node:x:${USER_UID}:${USER_GID}::/home/node:/bin/bash_ /etc/passwd \ && chown -R node:node /home/node \ && chown node:node /overleaf diff --git a/server-ce/Dockerfile b/server-ce/Dockerfile index 90019928c0..8c7d02606b 100644 --- a/server-ce/Dockerfile +++ b/server-ce/Dockerfile @@ -15,8 +15,11 @@ COPY server-ce/genScript.js server-ce/services.js /overleaf/ # Pre-install yarn via corepack so it is available at runtime for all users # ------------------------------------------------------------------------- +# Corepack setup, shared between all the images. +ENV PATH="/overleaf/node_modules/.bin:$PATH" ENV COREPACK_HOME=/opt/corepack -RUN corepack install +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 # Install yarn dependencies # ------------------------- diff --git a/server-ce/Dockerfile-base b/server-ce/Dockerfile-base index 731734ad7a..5b6912fd89 100644 --- a/server-ce/Dockerfile-base +++ b/server-ce/Dockerfile-base @@ -33,7 +33,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_22.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \ && apt-get update \ && apt-get install -y nodejs \ -&& corepack enable \ \ && rm -rf \ # We are adding a custom nginx config in the main Dockerfile. diff --git a/server-ce/test/Dockerfile b/server-ce/test/Dockerfile index 095766b688..5680b3e677 100644 --- a/server-ce/test/Dockerfile +++ b/server-ce/test/Dockerfile @@ -1,4 +1,11 @@ FROM node:24.14.1 + +# Corepack setup, shared between all the images. +ENV PATH="/overleaf/node_modules/.bin:$PATH" +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && echo \ "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ diff --git a/server-ce/test/Makefile b/server-ce/test/Makefile index 21c6dce4f4..cf3582e377 100644 --- a/server-ce/test/Makefile +++ b/server-ce/test/Makefile @@ -47,7 +47,7 @@ test-e2e-native-linux: docker compose -f docker-compose.yml -f docker-compose.native.yml up --no-log-prefix cypress test-e2e: - docker compose build host-admin + docker compose build host-admin e2e docker compose up --detach --wait host-admin docker compose up --detach --wait mongo docker compose up --no-log-prefix --exit-code-from=e2e e2e host-admin @@ -81,7 +81,7 @@ prefetch_default_compose: prefetch_default: prefetch_default_compose_build prefetch_default_compose_build: - docker compose build host-admin + docker compose build host-admin e2e prefetch: prefetch_custom prefetch_custom: prefetch_custom_compose_pull diff --git a/server-ce/test/docker-compose.yml b/server-ce/test/docker-compose.yml index 08b12d5a41..6d26e31f00 100644 --- a/server-ce/test/docker-compose.yml +++ b/server-ce/test/docker-compose.yml @@ -80,6 +80,10 @@ services: - $PWD:$PWD - $MONOREPO/libraries:$MONOREPO/libraries:ro - $MONOREPO/node_modules:$MONOREPO/node_modules:ro + - $MONOREPO/.yarn:$MONOREPO/.yarn:ro + - $MONOREPO/.yarnrc.yml:$MONOREPO/.yarnrc.yml:ro + - $MONOREPO/package.json:$MONOREPO/package.json:ro + - $MONOREPO/yarn.lock:$MONOREPO/yarn.lock:ro environment: MONOREPO: CYPRESS_SHARD: @@ -110,6 +114,10 @@ services: - $PWD:$PWD - $MONOREPO/libraries:$MONOREPO/libraries:ro - $MONOREPO/node_modules:$MONOREPO/node_modules:ro + - $MONOREPO/.yarn:$MONOREPO/.yarn:ro + - $MONOREPO/.yarnrc.yml:$MONOREPO/.yarnrc.yml:ro + - $MONOREPO/package.json:$MONOREPO/package.json:ro + - $MONOREPO/yarn.lock:$MONOREPO/yarn.lock:ro - /var/run/docker.sock:/var/run/docker.sock stop_grace_period: 0s environment: diff --git a/services/chat/Dockerfile b/services/chat/Dockerfile index 6f38f66e39..594fd47212 100644 --- a/services/chat/Dockerfile +++ b/services/chat/Dockerfile @@ -4,9 +4,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/chat +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/chat # Google Cloud Storage needs a writable $HOME/.config for resumable uploads # (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream) diff --git a/services/chat/Makefile b/services/chat/Makefile index cc4033f15e..5c9d69e652 100644 --- a/services/chat/Makefile +++ b/services/chat/Makefile @@ -35,6 +35,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) DOCKER_COMPOSE_TEST_ACCEPTANCE = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) @@ -52,11 +62,11 @@ clean: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) corepack yarn run --silent -RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) corepack yarn run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent +RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent SHELLCHECK_OPTS = \ --shell=bash \ @@ -123,7 +133,7 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) - $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit endif test_clean: test_unit_clean @@ -140,12 +150,12 @@ test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_accept test_acceptance_run: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance endif test_acceptance_run_debug: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance corepack yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance endif test_clean: test_acceptance_clean @@ -154,11 +164,11 @@ test_acceptance_clean: test_acceptance_pre_run: ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run endif benchmarks: - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance corepack yarn run benchmarks + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks build: docker build \ diff --git a/services/chat/docker-compose.ci.yml b/services/chat/docker-compose.ci.yml index bfe7314ced..23ebcb448a 100644 --- a/services/chat/docker-compose.ci.yml +++ b/services/chat/docker-compose.ci.yml @@ -21,7 +21,6 @@ services: condition: service_started test_acceptance: - build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: CI: diff --git a/services/chat/docker-compose.yml b/services/chat/docker-compose.yml index 9f43f52b70..32c167cd66 100644 --- a/services/chat/docker-compose.yml +++ b/services/chat/docker-compose.yml @@ -4,7 +4,10 @@ services: test_unit: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/chat/Dockerfile + target: base volumes: - .:/overleaf/services/chat - ../../node_modules:/overleaf/node_modules @@ -23,14 +26,17 @@ services: NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:unit + command: yarn run --silent test:unit user: node depends_on: mongo: condition: service_started test_acceptance: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/chat/Dockerfile + target: base volumes: - .:/overleaf/services/chat - ../../node_modules:/overleaf/node_modules @@ -55,7 +61,7 @@ services: mongo: condition: service_started entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:acceptance + command: yarn run --silent test:acceptance mongo: image: mongo:8.0.11 diff --git a/services/clsi/Dockerfile b/services/clsi/Dockerfile index bc4e0b8c3b..5abfd1a67c 100644 --- a/services/clsi/Dockerfile +++ b/services/clsi/Dockerfile @@ -4,9 +4,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/clsi +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/clsi COPY services/clsi/install_deps.sh /overleaf/services/clsi/ RUN chmod 0755 ./install_deps.sh && ./install_deps.sh ENTRYPOINT ["/bin/sh", "/entrypoint.sh"] diff --git a/services/clsi/Makefile b/services/clsi/Makefile index d1d01d5703..a35318c355 100644 --- a/services/clsi/Makefile +++ b/services/clsi/Makefile @@ -35,6 +35,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) DOCKER_COMPOSE_TEST_ACCEPTANCE = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) @@ -53,11 +63,11 @@ clean: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) corepack yarn run --silent -RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) corepack yarn run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent +RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent SHELLCHECK_OPTS = \ --shell=bash \ @@ -124,7 +134,7 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) - $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit endif test_clean: test_unit_clean @@ -141,12 +151,12 @@ test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_accept test_acceptance_run: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance endif test_acceptance_run_debug: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance corepack yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance endif test_clean: test_acceptance_clean @@ -157,11 +167,11 @@ test_acceptance_pre_run: docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc:3.9 docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc-staging:3.9 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run endif benchmarks: - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance corepack yarn run benchmarks + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks build: docker build \ diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml index e382ae6c91..9f206bfdd7 100644 --- a/services/clsi/docker-compose.ci.yml +++ b/services/clsi/docker-compose.ci.yml @@ -17,7 +17,6 @@ services: VITEST_NO_CACHE: true test_acceptance: - build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: CI: diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml index 1d9e50f730..b6590d4f1e 100644 --- a/services/clsi/docker-compose.yml +++ b/services/clsi/docker-compose.yml @@ -24,7 +24,7 @@ services: MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" - command: corepack yarn run --silent test:unit + command: yarn run --silent test:unit test_acceptance: build: @@ -64,7 +64,7 @@ services: depends_on: clsi-nginx: condition: service_started - command: corepack yarn run --silent test:acceptance + command: yarn run --silent test:acceptance clsi-nginx: image: nginx:1.28 diff --git a/services/contacts/Dockerfile b/services/contacts/Dockerfile index f12908d7b7..b01720ef31 100644 --- a/services/contacts/Dockerfile +++ b/services/contacts/Dockerfile @@ -4,9 +4,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/contacts +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/contacts # Google Cloud Storage needs a writable $HOME/.config for resumable uploads # (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream) diff --git a/services/contacts/Makefile b/services/contacts/Makefile index a7715240d4..cce1fadb19 100644 --- a/services/contacts/Makefile +++ b/services/contacts/Makefile @@ -35,6 +35,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) DOCKER_COMPOSE_TEST_ACCEPTANCE = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) @@ -52,11 +62,11 @@ clean: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/contacts/reports:/overleaf/services/contacts/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) corepack yarn run --silent -RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/contacts/reports:/overleaf/services/contacts/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) corepack yarn run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/contacts/reports:/overleaf/services/contacts/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent +RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/contacts/reports:/overleaf/services/contacts/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent SHELLCHECK_OPTS = \ --shell=bash \ @@ -123,7 +133,7 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) - $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit endif test_clean: test_unit_clean @@ -140,12 +150,12 @@ test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_accept test_acceptance_run: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance endif test_acceptance_run_debug: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance corepack yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance endif test_clean: test_acceptance_clean @@ -154,11 +164,11 @@ test_acceptance_clean: test_acceptance_pre_run: ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run endif benchmarks: - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance corepack yarn run benchmarks + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks build: docker build \ diff --git a/services/contacts/docker-compose.ci.yml b/services/contacts/docker-compose.ci.yml index a837490208..3a9128e545 100644 --- a/services/contacts/docker-compose.ci.yml +++ b/services/contacts/docker-compose.ci.yml @@ -21,7 +21,6 @@ services: condition: service_started test_acceptance: - build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: CI: diff --git a/services/contacts/docker-compose.yml b/services/contacts/docker-compose.yml index 5b6658904d..5e6cdfcee8 100644 --- a/services/contacts/docker-compose.yml +++ b/services/contacts/docker-compose.yml @@ -4,7 +4,10 @@ services: test_unit: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/contacts/Dockerfile + target: base volumes: - .:/overleaf/services/contacts - ../../node_modules:/overleaf/node_modules @@ -23,14 +26,17 @@ services: NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:unit + command: yarn run --silent test:unit user: node depends_on: mongo: condition: service_started test_acceptance: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/contacts/Dockerfile + target: base volumes: - .:/overleaf/services/contacts - ../../node_modules:/overleaf/node_modules @@ -55,7 +61,7 @@ services: mongo: condition: service_started entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:acceptance + command: yarn run --silent test:acceptance mongo: image: mongo:8.0.11 diff --git a/services/docstore/Dockerfile b/services/docstore/Dockerfile index c2e6a5ae1c..29615cbb3d 100644 --- a/services/docstore/Dockerfile +++ b/services/docstore/Dockerfile @@ -4,9 +4,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/docstore +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/docstore # Google Cloud Storage needs a writable $HOME/.config for resumable uploads # (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream) diff --git a/services/docstore/Makefile b/services/docstore/Makefile index f63dcf31e6..f6f8b2bc68 100644 --- a/services/docstore/Makefile +++ b/services/docstore/Makefile @@ -37,6 +37,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) DOCKER_COMPOSE_TEST_ACCEPTANCE = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) @@ -54,11 +64,11 @@ clean: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) corepack yarn run --silent -RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) corepack yarn run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent +RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent SHELLCHECK_OPTS = \ --shell=bash \ @@ -125,7 +135,7 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) - $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit endif test_clean: test_unit_clean @@ -142,12 +152,12 @@ test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_accept test_acceptance_run: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance endif test_acceptance_run_debug: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance corepack yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance endif test_clean: test_acceptance_clean @@ -156,11 +166,11 @@ test_acceptance_clean: test_acceptance_pre_run: ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run endif benchmarks: - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance corepack yarn run benchmarks + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks build: docker build \ diff --git a/services/docstore/docker-compose.ci.yml b/services/docstore/docker-compose.ci.yml index 02ce1ee453..2a1bdc7bd5 100644 --- a/services/docstore/docker-compose.ci.yml +++ b/services/docstore/docker-compose.ci.yml @@ -23,7 +23,6 @@ services: condition: service_started test_acceptance: - build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: CI: diff --git a/services/docstore/docker-compose.yml b/services/docstore/docker-compose.yml index 9bfb34e146..a3789640a1 100644 --- a/services/docstore/docker-compose.yml +++ b/services/docstore/docker-compose.yml @@ -4,7 +4,10 @@ services: test_unit: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/docstore/Dockerfile + target: base volumes: - .:/overleaf/services/docstore - ../../node_modules:/overleaf/node_modules @@ -24,14 +27,17 @@ services: NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:unit + command: yarn run --silent test:unit user: node depends_on: mongo: condition: service_started test_acceptance: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/docstore/Dockerfile + target: base volumes: - .:/overleaf/services/docstore - ../../node_modules:/overleaf/node_modules @@ -61,7 +67,7 @@ services: gcs: condition: service_healthy entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:acceptance + command: yarn run --silent test:acceptance mongo: image: mongo:8.0.11 diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index a6b3408192..d893c7b8b8 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -4,9 +4,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/document-updater +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/document-updater # Google Cloud Storage needs a writable $HOME/.config for resumable uploads # (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream) diff --git a/services/document-updater/Makefile b/services/document-updater/Makefile index d9f583638a..db34f4d3c7 100644 --- a/services/document-updater/Makefile +++ b/services/document-updater/Makefile @@ -38,6 +38,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) DOCKER_COMPOSE_TEST_ACCEPTANCE = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) @@ -55,11 +65,11 @@ clean: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) corepack yarn run --silent -RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) corepack yarn run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent +RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent SHELLCHECK_OPTS = \ --shell=bash \ @@ -126,7 +136,7 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) - $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit endif test_clean: test_unit_clean @@ -143,12 +153,12 @@ test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_accept test_acceptance_run: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance endif test_acceptance_run_debug: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance corepack yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance endif test_clean: test_acceptance_clean @@ -157,11 +167,11 @@ test_acceptance_clean: test_acceptance_pre_run: ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run endif benchmarks: - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance corepack yarn run benchmarks + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks build: docker build \ diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index 2f6ff56692..669d32eda4 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -28,7 +28,6 @@ services: condition: service_healthy test_acceptance: - build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: CI: diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index 3c50374612..7c577603c0 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -4,7 +4,10 @@ services: test_unit: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/document-updater/Dockerfile + target: base volumes: - .:/overleaf/services/document-updater - ../../node_modules:/overleaf/node_modules @@ -28,7 +31,7 @@ services: ANALYTICS_QUEUES_REDIS_HOST: redis_test ANALYTICS_QUEUES_BQ_SINK_REDIS_HOST: redis_test entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:unit + command: yarn run --silent test:unit user: node depends_on: mongo: @@ -37,7 +40,10 @@ services: condition: service_healthy test_acceptance: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/document-updater/Dockerfile + target: base volumes: - .:/overleaf/services/document-updater - ../../node_modules:/overleaf/node_modules @@ -69,7 +75,7 @@ services: redis_test: condition: service_healthy entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:acceptance + command: yarn run --silent test:acceptance redis_test: image: redis:7.4.8 diff --git a/services/filestore/Dockerfile b/services/filestore/Dockerfile index 8f9d1a071f..3a13379572 100644 --- a/services/filestore/Dockerfile +++ b/services/filestore/Dockerfile @@ -4,9 +4,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/filestore +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/filestore COPY services/filestore/install_deps.sh /overleaf/services/filestore/ RUN chmod 0755 ./install_deps.sh && ./install_deps.sh diff --git a/services/filestore/Makefile b/services/filestore/Makefile index d3cc6675f8..7687a353c5 100644 --- a/services/filestore/Makefile +++ b/services/filestore/Makefile @@ -34,6 +34,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) DOCKER_COMPOSE_TEST_ACCEPTANCE = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) @@ -52,11 +62,11 @@ clean: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) corepack yarn run --silent -RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) corepack yarn run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent +RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent SHELLCHECK_OPTS = \ --shell=bash \ @@ -123,7 +133,7 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) - $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit endif test_clean: test_unit_clean @@ -140,12 +150,12 @@ test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_accept test_acceptance_run: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance endif test_acceptance_run_debug: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance corepack yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance endif test_clean: test_acceptance_clean @@ -154,11 +164,11 @@ test_acceptance_clean: test_acceptance_pre_run: ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run endif benchmarks: - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance corepack yarn run benchmarks + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks build: docker build \ diff --git a/services/filestore/docker-compose.ci.yml b/services/filestore/docker-compose.ci.yml index 0d1d84fe35..64873a973f 100644 --- a/services/filestore/docker-compose.ci.yml +++ b/services/filestore/docker-compose.ci.yml @@ -21,7 +21,6 @@ services: VITEST_NO_CACHE: true test_acceptance: - build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: CI: diff --git a/services/filestore/docker-compose.yml b/services/filestore/docker-compose.yml index d5705c8e8c..4629a5a0cf 100644 --- a/services/filestore/docker-compose.yml +++ b/services/filestore/docker-compose.yml @@ -27,7 +27,7 @@ services: MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" - command: corepack yarn run --silent test:unit + command: yarn run --silent test:unit user: node test_acceptance: @@ -81,7 +81,7 @@ services: condition: service_completed_successfully gcs: condition: service_healthy - command: corepack yarn run --silent test:acceptance + command: yarn run --silent test:acceptance certs: build: diff --git a/services/history-v1/Dockerfile b/services/history-v1/Dockerfile index 7ed9e36d4c..ca71c38840 100644 --- a/services/history-v1/Dockerfile +++ b/services/history-v1/Dockerfile @@ -4,9 +4,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/history-v1 +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/history-v1 COPY services/history-v1/install_deps.sh /overleaf/services/history-v1/ RUN chmod 0755 ./install_deps.sh && ./install_deps.sh diff --git a/services/history-v1/Makefile b/services/history-v1/Makefile index 73b33b50d0..30b54180ee 100644 --- a/services/history-v1/Makefile +++ b/services/history-v1/Makefile @@ -40,6 +40,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) DOCKER_COMPOSE_TEST_ACCEPTANCE = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) @@ -57,11 +67,11 @@ clean: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) corepack yarn run --silent -RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) corepack yarn run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent +RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent SHELLCHECK_OPTS = \ --shell=bash \ @@ -128,7 +138,7 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) - $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit endif test_clean: test_unit_clean @@ -145,12 +155,12 @@ test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_accept test_acceptance_run: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance endif test_acceptance_run_debug: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance corepack yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance endif test_clean: test_acceptance_clean @@ -159,11 +169,11 @@ test_acceptance_clean: test_acceptance_pre_run: ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run endif benchmarks: - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance corepack yarn run benchmarks + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks build: docker build \ diff --git a/services/history-v1/docker-compose.ci.yml b/services/history-v1/docker-compose.ci.yml index fe837cb753..06ecdef7c2 100644 --- a/services/history-v1/docker-compose.ci.yml +++ b/services/history-v1/docker-compose.ci.yml @@ -31,7 +31,6 @@ services: condition: service_healthy test_acceptance: - build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: CI: diff --git a/services/history-v1/docker-compose.yml b/services/history-v1/docker-compose.yml index 0c3f6beaf5..cc4f8bd15d 100644 --- a/services/history-v1/docker-compose.yml +++ b/services/history-v1/docker-compose.yml @@ -34,7 +34,7 @@ services: ANALYTICS_QUEUES_REDIS_HOST: redis_test ANALYTICS_QUEUES_BQ_SINK_REDIS_HOST: redis_test entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:unit + command: yarn run --silent test:unit user: node depends_on: mongo: @@ -99,7 +99,7 @@ services: gcs: condition: service_healthy entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:acceptance + command: yarn run --silent test:acceptance redis_test: image: redis:7.4.8 diff --git a/services/notifications/Dockerfile b/services/notifications/Dockerfile index 28bb62b87a..bcca3d1d8f 100644 --- a/services/notifications/Dockerfile +++ b/services/notifications/Dockerfile @@ -4,9 +4,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/notifications +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/notifications # Google Cloud Storage needs a writable $HOME/.config for resumable uploads # (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream) diff --git a/services/notifications/Makefile b/services/notifications/Makefile index 774201abf5..d202504b80 100644 --- a/services/notifications/Makefile +++ b/services/notifications/Makefile @@ -36,6 +36,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) DOCKER_COMPOSE_TEST_ACCEPTANCE = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) @@ -53,11 +63,11 @@ clean: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) corepack yarn run --silent -RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) corepack yarn run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent +RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent SHELLCHECK_OPTS = \ --shell=bash \ @@ -124,7 +134,7 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) - $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit endif test_clean: test_unit_clean @@ -141,12 +151,12 @@ test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_accept test_acceptance_run: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance endif test_acceptance_run_debug: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance corepack yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance endif test_clean: test_acceptance_clean @@ -155,11 +165,11 @@ test_acceptance_clean: test_acceptance_pre_run: ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run endif benchmarks: - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance corepack yarn run benchmarks + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks build: docker build \ diff --git a/services/notifications/docker-compose.ci.yml b/services/notifications/docker-compose.ci.yml index 36af08b6cc..29bc06a1e2 100644 --- a/services/notifications/docker-compose.ci.yml +++ b/services/notifications/docker-compose.ci.yml @@ -23,7 +23,6 @@ services: condition: service_started test_acceptance: - build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: CI: diff --git a/services/notifications/docker-compose.yml b/services/notifications/docker-compose.yml index 6be5a04050..0361fea5fc 100644 --- a/services/notifications/docker-compose.yml +++ b/services/notifications/docker-compose.yml @@ -4,7 +4,10 @@ services: test_unit: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/notifications/Dockerfile + target: base volumes: - .:/overleaf/services/notifications - ../../node_modules:/overleaf/node_modules @@ -24,14 +27,17 @@ services: NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:unit + command: yarn run --silent test:unit user: node depends_on: mongo: condition: service_started test_acceptance: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/notifications/Dockerfile + target: base volumes: - .:/overleaf/services/notifications - ../../node_modules:/overleaf/node_modules @@ -57,7 +63,7 @@ services: mongo: condition: service_started entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:acceptance + command: yarn run --silent test:acceptance mongo: image: mongo:8.0.11 diff --git a/services/project-history/Dockerfile b/services/project-history/Dockerfile index dc68702875..26d469084f 100644 --- a/services/project-history/Dockerfile +++ b/services/project-history/Dockerfile @@ -4,9 +4,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/project-history +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/project-history # Google Cloud Storage needs a writable $HOME/.config for resumable uploads # (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream) diff --git a/services/project-history/Makefile b/services/project-history/Makefile index 92a11b365e..77fd893c75 100644 --- a/services/project-history/Makefile +++ b/services/project-history/Makefile @@ -39,6 +39,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) DOCKER_COMPOSE_TEST_ACCEPTANCE = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) @@ -56,11 +66,11 @@ clean: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) corepack yarn run --silent -RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) corepack yarn run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent +RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent SHELLCHECK_OPTS = \ --shell=bash \ @@ -127,7 +137,7 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) - $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit endif test_clean: test_unit_clean @@ -144,12 +154,12 @@ test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_accept test_acceptance_run: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance endif test_acceptance_run_debug: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance corepack yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance endif test_clean: test_acceptance_clean @@ -158,11 +168,11 @@ test_acceptance_clean: test_acceptance_pre_run: ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run endif benchmarks: - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance corepack yarn run benchmarks + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks build: docker build \ diff --git a/services/project-history/docker-compose.ci.yml b/services/project-history/docker-compose.ci.yml index 86cce42b3b..68679cc0dc 100644 --- a/services/project-history/docker-compose.ci.yml +++ b/services/project-history/docker-compose.ci.yml @@ -28,7 +28,6 @@ services: condition: service_healthy test_acceptance: - build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: CI: diff --git a/services/project-history/docker-compose.yml b/services/project-history/docker-compose.yml index 7e54d2166e..38f0a353d5 100644 --- a/services/project-history/docker-compose.yml +++ b/services/project-history/docker-compose.yml @@ -4,7 +4,10 @@ services: test_unit: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/project-history/Dockerfile + target: base volumes: - .:/overleaf/services/project-history - ../../node_modules:/overleaf/node_modules @@ -28,7 +31,7 @@ services: ANALYTICS_QUEUES_REDIS_HOST: redis_test ANALYTICS_QUEUES_BQ_SINK_REDIS_HOST: redis_test entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:unit + command: yarn run --silent test:unit user: node depends_on: mongo: @@ -37,7 +40,10 @@ services: condition: service_healthy test_acceptance: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/project-history/Dockerfile + target: base volumes: - .:/overleaf/services/project-history - ../../node_modules:/overleaf/node_modules @@ -69,7 +75,7 @@ services: redis_test: condition: service_healthy entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- - command: corepack yarn run --silent test:acceptance + command: yarn run --silent test:acceptance redis_test: image: redis:7.4.8 diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index 4d2bcb050a..2d309607c1 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -4,9 +4,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/real-time +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/real-time # Google Cloud Storage needs a writable $HOME/.config for resumable uploads # (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream) diff --git a/services/real-time/Makefile b/services/real-time/Makefile index d72091f9ba..d1ab7ffb02 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -35,6 +35,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) DOCKER_COMPOSE_TEST_ACCEPTANCE = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) @@ -52,11 +62,11 @@ clean: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) corepack yarn run --silent -RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) corepack yarn run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent +RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent SHELLCHECK_OPTS = \ --shell=bash \ @@ -123,7 +133,7 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) - $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit endif test_clean: test_unit_clean @@ -140,12 +150,12 @@ test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_accept test_acceptance_run: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance endif test_acceptance_run_debug: ifneq (,$(wildcard test/acceptance)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance corepack yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance endif test_clean: test_acceptance_clean @@ -154,11 +164,11 @@ test_acceptance_clean: test_acceptance_pre_run: ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run endif benchmarks: - $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance corepack yarn run benchmarks + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks build: docker build \ diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index b3e07b0a17..6b57013eb2 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -26,7 +26,6 @@ services: condition: service_healthy test_acceptance: - build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: CI: diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 0e592dc288..678d96e22d 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -4,7 +4,10 @@ services: test_unit: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/real-time/Dockerfile + target: base volumes: - .:/overleaf/services/real-time - ../../node_modules:/overleaf/node_modules @@ -26,14 +29,17 @@ services: QUEUES_REDIS_HOST: redis_test ANALYTICS_QUEUES_REDIS_HOST: redis_test ANALYTICS_QUEUES_BQ_SINK_REDIS_HOST: redis_test - command: corepack yarn run --silent test:unit + command: yarn run --silent test:unit user: node depends_on: redis_test: condition: service_healthy test_acceptance: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/real-time/Dockerfile + target: base volumes: - .:/overleaf/services/real-time - ../../node_modules:/overleaf/node_modules @@ -60,7 +66,7 @@ services: depends_on: redis_test: condition: service_healthy - command: corepack yarn run --silent test:acceptance + command: yarn run --silent test:acceptance redis_test: image: redis:7.4.8 diff --git a/services/web/Dockerfile b/services/web/Dockerfile index 9046917eb3..47ed96ed0b 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -3,9 +3,13 @@ FROM node:24.14.1 AS base -WORKDIR /overleaf/services/web +# Corepack setup, shared between all the images. ENV PATH="/overleaf/node_modules/.bin:$PATH" -RUN corepack enable +ENV COREPACK_HOME=/opt/corepack +RUN corepack enable && corepack install -g yarn@4.14.1 +ENV COREPACK_ENABLE_NETWORK=0 + +WORKDIR /overleaf/services/web # Google Cloud Storage needs a writable $HOME/.config for resumable uploads # (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream) diff --git a/services/web/Makefile b/services/web/Makefile index 0790947f17..64c5cf2ecd 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -29,6 +29,16 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker compose ${DOCKER_COMPOSE_FLAGS} +# Common flags for docker compose run. +# --rm Tear down test runner container +# --build Make sure that we use the latest base image +# --quiet-build Hide build output +ifeq ($(CI),true) + DC_RUN_FLAGS := --rm +else + DC_RUN_FLAGS := --rm --build --quiet-build +endif + MODULE_DIRS := $(shell find modules -mindepth 1 -maxdepth 1 -type d -not -name '.git' ) MODULE_MAKEFILES := $(MODULE_DIRS:=/Makefile) MODULE_NAME=$(shell basename $(MODULE)) @@ -107,11 +117,11 @@ test_module: test_unit_module test_acceptance_module test_unit: test_unit_all test_unit_all: export COMPOSE_PROJECT_NAME=unit_test_all_$(BUILD_DIR_NAME) test_unit_all: mongo_migrations_for_tests - $(DOCKER_COMPOSE) run --rm test_unit yarn run test:unit:all + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_unit yarn run test:unit:all test_unit_all_silent: export COMPOSE_PROJECT_NAME=unit_test_all_$(BUILD_DIR_NAME) test_unit_all_silent: mongo_migrations_for_tests - $(DOCKER_COMPOSE) run --rm test_unit yarn run test:unit:all:silent + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_unit yarn run test:unit:all:silent test_unit_app: export COMPOSE_PROJECT_NAME=unit_test_$(BUILD_DIR_NAME) test_unit_app: mongo_migrations_for_tests @@ -120,16 +130,16 @@ test_unit_app: mongo_migrations_for_tests test_unit_parallel: export JUNIT_ROOT_SUITE_NAME = ESM unit tests - parallel test_unit_parallel: export COMPOSE_PROJECT_NAME=unit_test_parallel_$(BUILD_DIR_NAME) test_unit_parallel: mongo_migrations_for_tests - $(DOCKER_COMPOSE) run --rm test_unit yarn run test:unit:parallel + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_unit yarn run test:unit:parallel test_unit_sequential: export JUNIT_ROOT_SUITE_NAME = ESM unit tests - sequential test_unit_sequential: export COMPOSE_PROJECT_NAME=unit_test_sequential_$(BUILD_DIR_NAME) test_unit_sequential: mongo_migrations_for_tests - $(DOCKER_COMPOSE) run --rm test_unit yarn run test:unit:sequential + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_unit yarn run test:unit:sequential test_unit_watch: export COMPOSE_PROJECT_NAME=unit_test_watch_$(BUILD_DIR_NAME) test_unit_watch: mongo_migrations_for_tests - $(DOCKER_COMPOSE) run --rm test_unit yarn run test:unit:watch + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_unit yarn run test:unit:watch TEST_SUITES = $(sort $(filter-out \ $(wildcard test/unit/src/helpers/*), \ @@ -152,7 +162,7 @@ test_unit_app_parallel_gnu_make: $(TEST_SUITES) test_unit_app_parallel_gnu_make_docker: export COMPOSE_PROJECT_NAME = \ unit_test_parallel_make_$(BUILD_DIR_NAME) test_unit_app_parallel_gnu_make_docker: mongo_migrations_for_tests - $(DOCKER_COMPOSE) run --rm test_unit \ + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_unit \ make test_unit_app_parallel_gnu_make --output-sync -j $(J) TEST_UNIT_MODULES = $(MODULE_DIRS:=/test_unit) @@ -163,7 +173,7 @@ test_unit_module: mongo_migrations_for_tests $(MAKE) modules/$(MODULE_NAME)/test_unit mongo_migrations_for_tests: - $(DOCKER_COMPOSE) run --rm --workdir /overleaf/tools/migrations test_unit yarn run migrations migrate -t saas + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) --workdir /overleaf/tools/migrations test_unit yarn run migrations migrate -t saas # # Frontend tests @@ -171,7 +181,7 @@ mongo_migrations_for_tests: test_frontend_jsdom: export JUNIT_ROOT_SUITE_NAME = JSDOM frontend tests test_frontend_jsdom: - COMPOSE_PROJECT_NAME=frontend_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) run --rm test_frontend_jsdom + COMPOSE_PROJECT_NAME=frontend_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_frontend_jsdom COMPOSE_PROJECT_NAME=frontend_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down -v -t 0 test_frontend: test_frontend_jsdom test_frontend_ct @@ -198,7 +208,7 @@ $(TEST_FRONTEND_CT_TARGETS): export USER_UID=$(shell id -u) $(TEST_FRONTEND_CT_TARGETS): export USER_GID=$(shell id -g) test_frontend_ct_ui: - docker compose -f docker-compose.cypress.yml run --rm cypress run cypress:open-ct -- --browser chrome + docker compose -f docker-compose.cypress.yml run $(DC_RUN_FLAGS) cypress run cypress:open-ct -- --browser chrome # # Writefull tests @@ -206,7 +216,7 @@ test_frontend_ct_ui: test_writefull: export JUNIT_ROOT_SUITE_NAME = Writefull tests test_writefull: - COMPOSE_PROJECT_NAME=writefull_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) run --rm test_writefull + COMPOSE_PROJECT_NAME=writefull_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_writefull COMPOSE_PROJECT_NAME=writefull_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down -v -t 0 # Note: The below cypress targets are for CI only @@ -238,7 +248,7 @@ test_frontend_ct_editor_visual: export CYPRESS_RESULTS=./cypress/results/editor_ test_frontend_ct_editor_visual: export CYPRESS_SPEC_PATTERN=./test/frontend/features/source-editor/components/codemirror-editor-visual*.spec.{js,jsx,ts,tsx} $(TEST_FRONTEND_CT_VARIANTS): - COMPOSE_PROJECT_NAME=$@_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) run --rm test_frontend_ct + COMPOSE_PROJECT_NAME=$@_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_frontend_ct COMPOSE_PROJECT_NAME=$@_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down -v -t 0 # @@ -275,7 +285,7 @@ test_acceptance_app_server_pro: export COMPOSE_PROJECT_NAME=acceptance_test_serv test_acceptance_app_server_pro: export OVERLEAF_CONFIG=$(CFG_SERVER_PRO) $(TEST_ACCEPTANCE_APP): - $(DOCKER_COMPOSE) run --rm $(TEST_ACCEPTANCE_DEBUG_PORT) test_acceptance $(TEST_ACCEPTANCE_YARN_SCRIPT) + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) $(TEST_ACCEPTANCE_DEBUG_PORT) test_acceptance $(TEST_ACCEPTANCE_YARN_SCRIPT) $(DOCKER_COMPOSE) down -v -t 0 # We are using _make magic_ for turning these file-targets into calls to @@ -349,7 +359,7 @@ test_acceptance_module_maybe_in_server_pro: export BASE_CONFIG=$(CFG_SERVER_PRO) $(TEST_ACCEPTANCE_MODULE_MAYBE_IN): test_acceptance_module_maybe_in_%: $(MAKE) $(shell \ OVERLEAF_CONFIG=$(BASE_CONFIG) \ - $(DOCKER_COMPOSE) run --rm test_unit \ + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_unit \ yarn node test/acceptance/getModuleTargets test_acceptance_$* \ | grep -e /$(MODULE_NAME)/ || echo test_acceptance_module_noop LABEL=$* \ ) @@ -427,7 +437,7 @@ TEST_ACCEPTANCE_MODULES_MERGED_VARIANTS = \ test_acceptance_modules_merged_server_pro \ $(TEST_ACCEPTANCE_MODULES_MERGED_VARIANTS): - $(DOCKER_COMPOSE) run --rm test_acceptance yarn exec make test_acceptance_modules_merged_inner + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_acceptance yarn exec make test_acceptance_modules_merged_inner $(DOCKER_COMPOSE) down -v -t 0 # outer loop for running saas tests in parallel @@ -452,7 +462,7 @@ $(TEST_ACCEPTANCE_MODULES_MERGED_SPLIT_SAAS): export BASE_CONFIG = $(CFG_SAAS) $(TEST_ACCEPTANCE_MODULES_MERGED_SPLIT_SAAS): export JUNIT_ROOT_SUITE_NAME = SaaS modules acceptance tests $(TEST_ACCEPTANCE_MODULES_MERGED_SPLIT_SAAS): test_acceptance_modules_merged_saas_%: - $(DOCKER_COMPOSE) run --rm test_acceptance yarn exec make test_acceptance_modules_merged_inner_$* + $(DOCKER_COMPOSE) run $(DC_RUN_FLAGS) test_acceptance yarn exec make test_acceptance_modules_merged_inner_$* $(DOCKER_COMPOSE) down -v -t 0 test_acceptance_modules: $(TEST_ACCEPTANCE_MODULES_MERGED_VARIANTS) @@ -479,8 +489,8 @@ ci: # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/web/data/reports:/overleaf/services/web/data/reports --volume $(MONOREPO)/node_modules/.cache/:/overleaf/node_modules/.cache/ -w /overleaf $(IMAGE_CI) yarn run --silent -RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(PWD) -e PATH=$(MONOREPO)/node_modules/.bin:$$PATH --user node node:24.14.1 corepack yarn run --silent -RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) --user node node:24.14.1 corepack yarn run --silent +RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent +RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent ORG_PATH = /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_LINT_FORMAT ?= \ diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml index cbfff07bec..55e60ce1f8 100644 --- a/services/web/docker-compose.yml +++ b/services/web/docker-compose.yml @@ -3,7 +3,10 @@ volumes: services: test_unit: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/web/Dockerfile + target: base volumes: - .:/overleaf/services/web - ../../node_modules:/overleaf/node_modules @@ -30,7 +33,7 @@ services: NODE_OPTIONS: "--unhandled-rejections=strict" REDIS_HOST: redis_test MONOREPO: - entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- corepack + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: yarn run --silent test:unit:app user: node depends_on: @@ -38,7 +41,10 @@ services: - redis_test test_acceptance: - image: node:24.14.1 + build: + context: ../.. + dockerfile: services/web/Dockerfile + target: base volumes: - .:/overleaf/services/web - ../../node_modules:/overleaf/node_modules @@ -75,7 +81,7 @@ services: - mongo - saml - ldap - entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- corepack + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: yarn run --silent test:acceptance:app test_frontend_jsdom: @@ -98,7 +104,6 @@ services: MOCHA_GREP: ${MOCHA_GREP} NODE_OPTIONS: "--unhandled-rejections=strict" VERBOSE_LOGGING: - entrypoint: corepack command: yarn run --silent test:frontend user: node @@ -141,7 +146,6 @@ services: JUNIT_ROOT_SUITE_NAME: NODE_OPTIONS: "--unhandled-rejections=strict" VERBOSE_LOGGING: - entrypoint: corepack command: yarn run --silent test:writefull user: node