From 79eabd511e5ed59ed8063d1c525c5c694ae2f74d Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 14 Oct 2025 14:46:45 +0200 Subject: [PATCH] [monorepo] use content hash as docker image cache key (#29046) * [monorepo] remove docker-repos flag from build scripts * [monorepo] use content hash as docker image cache key * [packer] jenkins-worker: populate build cache for all services * [v1] adopt smarter docker caching * [latexqc] adopt smarter docker caching * [monorepo] refresh docker cache in Jenkins * [packer] jenkins-worker: increase disk size * [monorepo] run jenkins_docker_build_cache as Jenkins user for v1 * [monorepo] define MONOREPO at the top of all the Makefiles * [monorepo] add --build-arg BUILDKIT_INLINE_CACHE=1 everywhere GitOrigin-RevId: 8110c6e68727a52ea80f3ec71711d30e17d69499 --- .../access-token-encryptor/buildscript.txt | 1 - libraries/fetch-utils/buildscript.txt | 1 - libraries/logger/buildscript.txt | 1 - libraries/metrics/buildscript.txt | 1 - libraries/mongo-utils/buildscript.txt | 1 - libraries/o-error/buildscript.txt | 1 - libraries/object-persistor/buildscript.txt | 1 - .../overleaf-editor-core/buildscript.txt | 1 - libraries/promise-utils/buildscript.txt | 1 - libraries/ranges-tracker/buildscript.txt | 1 - libraries/redis-wrapper/buildscript.txt | 1 - libraries/settings/buildscript.txt | 1 - libraries/stream-utils/buildscript.txt | 1 - libraries/validation-tools/buildscript.txt | 1 - server-ce/Makefile | 9 ++++ server-ce/test/Jenkinsfile | 1 + services/chat/Makefile | 38 ++++++++++------ services/chat/buildscript.txt | 1 - services/clsi/Makefile | 39 +++++++++++------ services/clsi/buildscript.txt | 1 - services/contacts/Makefile | 38 ++++++++++------ services/contacts/buildscript.txt | 1 - services/docstore/Makefile | 41 ++++++++++++------ services/docstore/buildscript.txt | 1 - services/document-updater/Makefile | 42 ++++++++++++------ services/document-updater/buildscript.txt | 1 - services/filestore/Makefile | 39 +++++++++++------ services/filestore/buildscript.txt | 1 - services/git-bridge/Jenkinsfile | 1 + services/git-bridge/Makefile | 25 +++++++++-- services/history-v1/Makefile | 43 +++++++++++++------ services/history-v1/buildscript.txt | 1 - services/notifications/Makefile | 40 +++++++++++------ services/notifications/buildscript.txt | 1 - services/project-history/Makefile | 42 ++++++++++++------ services/project-history/buildscript.txt | 1 - services/real-time/Makefile | 38 ++++++++++------ services/real-time/buildscript.txt | 1 - services/web/Jenkinsfile | 2 +- services/web/Makefile | 39 +++++++++++------ 40 files changed, 341 insertions(+), 160 deletions(-) diff --git a/libraries/access-token-encryptor/buildscript.txt b/libraries/access-token-encryptor/buildscript.txt index acc186c0e0..da713de484 100644 --- a/libraries/access-token-encryptor/buildscript.txt +++ b/libraries/access-token-encryptor/buildscript.txt @@ -1,6 +1,5 @@ access-token-encryptor --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/fetch-utils/buildscript.txt b/libraries/fetch-utils/buildscript.txt index 0c65930010..37eef409c8 100644 --- a/libraries/fetch-utils/buildscript.txt +++ b/libraries/fetch-utils/buildscript.txt @@ -1,6 +1,5 @@ fetch-utils --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/logger/buildscript.txt b/libraries/logger/buildscript.txt index c1f0463906..b794b66e02 100644 --- a/libraries/logger/buildscript.txt +++ b/libraries/logger/buildscript.txt @@ -1,6 +1,5 @@ logger --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/metrics/buildscript.txt b/libraries/metrics/buildscript.txt index 6389b01707..6329b90406 100644 --- a/libraries/metrics/buildscript.txt +++ b/libraries/metrics/buildscript.txt @@ -1,6 +1,5 @@ metrics --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/mongo-utils/buildscript.txt b/libraries/mongo-utils/buildscript.txt index ea865b559f..8d5ed1f6d7 100644 --- a/libraries/mongo-utils/buildscript.txt +++ b/libraries/mongo-utils/buildscript.txt @@ -1,6 +1,5 @@ mongo-utils --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/o-error/buildscript.txt b/libraries/o-error/buildscript.txt index da386902df..e927bd5285 100644 --- a/libraries/o-error/buildscript.txt +++ b/libraries/o-error/buildscript.txt @@ -1,6 +1,5 @@ o-error --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/object-persistor/buildscript.txt b/libraries/object-persistor/buildscript.txt index 9531a27981..7dd7ac5e88 100644 --- a/libraries/object-persistor/buildscript.txt +++ b/libraries/object-persistor/buildscript.txt @@ -1,6 +1,5 @@ object-persistor --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/overleaf-editor-core/buildscript.txt b/libraries/overleaf-editor-core/buildscript.txt index 0736aa22f7..018697f1ec 100644 --- a/libraries/overleaf-editor-core/buildscript.txt +++ b/libraries/overleaf-editor-core/buildscript.txt @@ -1,6 +1,5 @@ overleaf-editor-core --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/promise-utils/buildscript.txt b/libraries/promise-utils/buildscript.txt index 94e4e281c0..0eba913085 100644 --- a/libraries/promise-utils/buildscript.txt +++ b/libraries/promise-utils/buildscript.txt @@ -1,6 +1,5 @@ promise-utils --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/ranges-tracker/buildscript.txt b/libraries/ranges-tracker/buildscript.txt index 0e91b68911..3cf85f0912 100644 --- a/libraries/ranges-tracker/buildscript.txt +++ b/libraries/ranges-tracker/buildscript.txt @@ -1,6 +1,5 @@ ranges-tracker --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/redis-wrapper/buildscript.txt b/libraries/redis-wrapper/buildscript.txt index 5e40f53ece..a6d623fde9 100644 --- a/libraries/redis-wrapper/buildscript.txt +++ b/libraries/redis-wrapper/buildscript.txt @@ -1,6 +1,5 @@ redis-wrapper --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/settings/buildscript.txt b/libraries/settings/buildscript.txt index 1b86dc9ad5..63025804e4 100644 --- a/libraries/settings/buildscript.txt +++ b/libraries/settings/buildscript.txt @@ -1,6 +1,5 @@ settings --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/stream-utils/buildscript.txt b/libraries/stream-utils/buildscript.txt index 0809298b64..d21491c6b6 100644 --- a/libraries/stream-utils/buildscript.txt +++ b/libraries/stream-utils/buildscript.txt @@ -1,6 +1,5 @@ stream-utils --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/libraries/validation-tools/buildscript.txt b/libraries/validation-tools/buildscript.txt index 22de13c6c6..edfacd3cb2 100644 --- a/libraries/validation-tools/buildscript.txt +++ b/libraries/validation-tools/buildscript.txt @@ -1,6 +1,5 @@ validation-tools --dependencies=None ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/server-ce/Makefile b/server-ce/Makefile index 5ce972154e..56f16362d4 100644 --- a/server-ce/Makefile +++ b/server-ce/Makefile @@ -16,6 +16,15 @@ all: build-base build-community clean: -docker rmi --force $(OVERLEAF_BASE_TAG) $(OVERLEAF_TAG) +refresh-cache: refresh-cache-branch +refresh-cache-branch: + docker inspect $(OVERLEAF_BASE_BRANCH) > /dev/null && docker pull $(OVERLEAF_BASE_BRANCH) || true + docker inspect $(OVERLEAF_BRANCH) > /dev/null && docker pull $(OVERLEAF_BRANCH) || true +refresh-cache: refresh-cache-latest +refresh-cache-latest: + docker inspect $(OVERLEAF_BASE_LATEST) > /dev/null && docker pull $(OVERLEAF_BASE_LATEST) || true + docker inspect $(OVERLEAF_LATEST) > /dev/null && docker pull $(OVERLEAF_LATEST) || true + build-base: cp .dockerignore $(MONOREPO_ROOT) docker build \ diff --git a/server-ce/test/Jenkinsfile b/server-ce/test/Jenkinsfile index b0f45a3fd8..ed19b218fa 100644 --- a/server-ce/test/Jenkinsfile +++ b/server-ce/test/Jenkinsfile @@ -104,6 +104,7 @@ pipeline { } } dir('copybara/public/repo/server-ce') { + sh 'make refresh-cache -j2' retry(count: 3) { sh 'make build-base' } diff --git a/services/chat/Makefile b/services/chat/Makefile index 58b95620cf..790c477540 100644 --- a/services/chat/Makefile +++ b/services/chat/Makefile @@ -6,6 +6,23 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = chat BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +HERE=$(shell pwd) +export MONOREPO ?= $(shell cd ../../ && pwd) +IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME)-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/mongo-utils/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/services/chat/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ @@ -23,19 +40,17 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) clean: - -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi $(IMAGE_CI) + -docker rmi $(IMAGE_REPO_FINAL) -$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes -rm -rf reports/ -HERE=$(shell pwd) -export MONOREPO ?= $(shell cd ../../ && pwd) # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports $(IMAGE_CI) npm run --silent # Same but from the top of the monorepo RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent @@ -128,11 +143,10 @@ build: docker build \ --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --tag $(IMAGE_CI) \ + --tag $(IMAGE_CACHE) \ + --tag $(IMAGE_REPO_FINAL) \ + --cache-from $(IMAGE_CACHE) \ --file Dockerfile \ ../.. @@ -140,10 +154,10 @@ tar: $(DOCKER_COMPOSE) up tar push: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker push $(IMAGE_REPO_FINAL) push_branch: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) + docker push $(IMAGE_CACHE) .PHONY: clean \ format format_fix \ diff --git a/services/chat/buildscript.txt b/services/chat/buildscript.txt index 0354503209..53ef1300b8 100644 --- a/services/chat/buildscript.txt +++ b/services/chat/buildscript.txt @@ -1,6 +1,5 @@ chat --dependencies=mongo ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/services/clsi/Makefile b/services/clsi/Makefile index 8f90e0dfca..c8fce8ba22 100644 --- a/services/clsi/Makefile +++ b/services/clsi/Makefile @@ -6,6 +6,24 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = clsi BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +HERE=$(shell pwd) +export MONOREPO ?= $(shell cd ../../ && pwd) +IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME)-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/promise-utils/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/libraries/stream-utils/package.json \ + $(MONOREPO)/services/clsi/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ @@ -23,20 +41,18 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) clean: - -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi $(IMAGE_CI) + -docker rmi $(IMAGE_REPO_FINAL) -$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes -rm -rf reports/ -git clean -dfX cache compiles output -HERE=$(shell pwd) -export MONOREPO ?= $(shell cd ../../ && pwd) # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports $(IMAGE_CI) npm run --silent # Same but from the top of the monorepo RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent @@ -129,11 +145,10 @@ build: docker build \ --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --tag $(IMAGE_CI) \ + --tag $(IMAGE_CACHE) \ + --tag $(IMAGE_REPO_FINAL) \ + --cache-from $(IMAGE_CACHE) \ --file Dockerfile \ ../.. @@ -141,10 +156,10 @@ tar: $(DOCKER_COMPOSE) up tar push: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker push $(IMAGE_REPO_FINAL) push_branch: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) + docker push $(IMAGE_CACHE) .PHONY: clean \ format format_fix \ diff --git a/services/clsi/buildscript.txt b/services/clsi/buildscript.txt index b2bf4bbd05..cac5f8e3c3 100644 --- a/services/clsi/buildscript.txt +++ b/services/clsi/buildscript.txt @@ -1,7 +1,6 @@ clsi --data-dirs=cache,compiles,output --dependencies= ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER="tex",SANDBOXED_COMPILES="true",SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output --env-pass-through= --esmock-loader=False diff --git a/services/contacts/Makefile b/services/contacts/Makefile index 2e3192dd9c..201713f394 100644 --- a/services/contacts/Makefile +++ b/services/contacts/Makefile @@ -6,6 +6,23 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = contacts BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +HERE=$(shell pwd) +export MONOREPO ?= $(shell cd ../../ && pwd) +IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME)-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/mongo-utils/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/services/contacts/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ @@ -23,19 +40,17 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) clean: - -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi $(IMAGE_CI) + -docker rmi $(IMAGE_REPO_FINAL) -$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes -rm -rf reports/ -HERE=$(shell pwd) -export MONOREPO ?= $(shell cd ../../ && pwd) # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/contacts/reports:/overleaf/services/contacts/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/contacts/reports:/overleaf/services/contacts/reports $(IMAGE_CI) npm run --silent # Same but from the top of the monorepo RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent @@ -128,11 +143,10 @@ build: docker build \ --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --tag $(IMAGE_CI) \ + --tag $(IMAGE_CACHE) \ + --tag $(IMAGE_REPO_FINAL) \ + --cache-from $(IMAGE_CACHE) \ --file Dockerfile \ ../.. @@ -140,10 +154,10 @@ tar: $(DOCKER_COMPOSE) up tar push: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker push $(IMAGE_REPO_FINAL) push_branch: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) + docker push $(IMAGE_CACHE) .PHONY: clean \ format format_fix \ diff --git a/services/contacts/buildscript.txt b/services/contacts/buildscript.txt index 4ae0783af7..ba2386c5b1 100644 --- a/services/contacts/buildscript.txt +++ b/services/contacts/buildscript.txt @@ -1,6 +1,5 @@ contacts --dependencies=mongo ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=True diff --git a/services/docstore/Makefile b/services/docstore/Makefile index 2dce268782..b7c8ddda5a 100644 --- a/services/docstore/Makefile +++ b/services/docstore/Makefile @@ -6,6 +6,26 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = docstore BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +HERE=$(shell pwd) +export MONOREPO ?= $(shell cd ../../ && pwd) +IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME)-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/mongo-utils/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/object-persistor/package.json \ + $(MONOREPO)/libraries/promise-utils/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/libraries/stream-utils/package.json \ + $(MONOREPO)/services/docstore/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ @@ -23,19 +43,17 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) clean: - -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi $(IMAGE_CI) + -docker rmi $(IMAGE_REPO_FINAL) -$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes -rm -rf reports/ -HERE=$(shell pwd) -export MONOREPO ?= $(shell cd ../../ && pwd) # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports $(IMAGE_CI) npm run --silent # Same but from the top of the monorepo RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent @@ -128,11 +146,10 @@ build: docker build \ --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --tag $(IMAGE_CI) \ + --tag $(IMAGE_CACHE) \ + --tag $(IMAGE_REPO_FINAL) \ + --cache-from $(IMAGE_CACHE) \ --file Dockerfile \ ../.. @@ -140,10 +157,10 @@ tar: $(DOCKER_COMPOSE) up tar push: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker push $(IMAGE_REPO_FINAL) push_branch: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) + docker push $(IMAGE_CACHE) .PHONY: clean \ format format_fix \ diff --git a/services/docstore/buildscript.txt b/services/docstore/buildscript.txt index 2390695ec3..f3ca0cf104 100644 --- a/services/docstore/buildscript.txt +++ b/services/docstore/buildscript.txt @@ -1,6 +1,5 @@ docstore --dependencies=mongo,gcs ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/services/document-updater/Makefile b/services/document-updater/Makefile index e23d5ce1ae..85c1acf5c0 100644 --- a/services/document-updater/Makefile +++ b/services/document-updater/Makefile @@ -6,6 +6,27 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = document-updater BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +HERE=$(shell pwd) +export MONOREPO ?= $(shell cd ../../ && pwd) +IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME)-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/mongo-utils/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/overleaf-editor-core/package.json \ + $(MONOREPO)/libraries/promise-utils/package.json \ + $(MONOREPO)/libraries/ranges-tracker/package.json \ + $(MONOREPO)/libraries/redis-wrapper/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/services/document-updater/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ @@ -23,19 +44,17 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) clean: - -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi $(IMAGE_CI) + -docker rmi $(IMAGE_REPO_FINAL) -$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes -rm -rf reports/ -HERE=$(shell pwd) -export MONOREPO ?= $(shell cd ../../ && pwd) # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports $(IMAGE_CI) npm run --silent # Same but from the top of the monorepo RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent @@ -128,11 +147,10 @@ build: docker build \ --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --tag $(IMAGE_CI) \ + --tag $(IMAGE_CACHE) \ + --tag $(IMAGE_REPO_FINAL) \ + --cache-from $(IMAGE_CACHE) \ --file Dockerfile \ ../.. @@ -140,10 +158,10 @@ tar: $(DOCKER_COMPOSE) up tar push: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker push $(IMAGE_REPO_FINAL) push_branch: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) + docker push $(IMAGE_CACHE) .PHONY: clean \ format format_fix \ diff --git a/services/document-updater/buildscript.txt b/services/document-updater/buildscript.txt index a5c62f28ea..728470fada 100644 --- a/services/document-updater/buildscript.txt +++ b/services/document-updater/buildscript.txt @@ -1,6 +1,5 @@ document-updater --dependencies=mongo,redis ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/services/filestore/Makefile b/services/filestore/Makefile index 3ac96011c6..14bb97be87 100644 --- a/services/filestore/Makefile +++ b/services/filestore/Makefile @@ -6,6 +6,24 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = filestore BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +HERE=$(shell pwd) +export MONOREPO ?= $(shell cd ../../ && pwd) +IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME)-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/object-persistor/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/libraries/stream-utils/package.json \ + $(MONOREPO)/services/filestore/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ @@ -23,20 +41,18 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) clean: - -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi $(IMAGE_CI) + -docker rmi $(IMAGE_REPO_FINAL) -$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes -rm -rf reports/ -git clean -dfX uploads template_files -HERE=$(shell pwd) -export MONOREPO ?= $(shell cd ../../ && pwd) # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports $(IMAGE_CI) npm run --silent # Same but from the top of the monorepo RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent @@ -129,11 +145,10 @@ build: docker build \ --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --tag $(IMAGE_CI) \ + --tag $(IMAGE_CACHE) \ + --tag $(IMAGE_REPO_FINAL) \ + --cache-from $(IMAGE_CACHE) \ --file Dockerfile \ ../.. @@ -141,10 +156,10 @@ tar: $(DOCKER_COMPOSE) up tar push: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker push $(IMAGE_REPO_FINAL) push_branch: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) + docker push $(IMAGE_CACHE) .PHONY: clean \ format format_fix \ diff --git a/services/filestore/buildscript.txt b/services/filestore/buildscript.txt index 653cb7e96e..810b3a1107 100644 --- a/services/filestore/buildscript.txt +++ b/services/filestore/buildscript.txt @@ -1,7 +1,6 @@ filestore --data-dirs=uploads,template_files --dependencies=s3,gcs ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_STORAGE_CLASS=REDUCED_REDUNDANCY,AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_USER_FILES_DEK_BUCKET_NAME=fake-user-files-dek,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files --env-pass-through= --esmock-loader=False diff --git a/services/git-bridge/Jenkinsfile b/services/git-bridge/Jenkinsfile index 036c4ee485..c8fa9ff7c5 100644 --- a/services/git-bridge/Jenkinsfile +++ b/services/git-bridge/Jenkinsfile @@ -22,6 +22,7 @@ pipeline { stage('Build') { steps { dir('services/git-bridge') { + sh 'make refresh_cache -j2' retry(count: 3) { sh 'make docker_build_base' } diff --git a/services/git-bridge/Makefile b/services/git-bridge/Makefile index edd8ab392f..fbc350ba01 100644 --- a/services/git-bridge/Makefile +++ b/services/git-bridge/Makefile @@ -28,10 +28,22 @@ $(MVN_TARGET): $(shell find src -type f) pom.xml build: $(MVN_TARGET) docker_build_base: - docker build --tag $(IMAGE_CI) --target base . + docker build --pull \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from $(IMAGE_REPO_BRANCH) \ + --cache-from $(IMAGE_REPO_MAIN) \ + --tag $(IMAGE_CI) \ + --target base \ + . docker_build: - docker build --tag $(IMAGE_REPO_BRANCH) --tag $(IMAGE_REPO_FINAL) . + docker build --pull \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from $(IMAGE_REPO_BRANCH) \ + --cache-from $(IMAGE_REPO_MAIN) \ + --tag $(IMAGE_REPO_BRANCH) \ + --tag $(IMAGE_REPO_FINAL) \ + . format: mvn $(MVN_OPTS) com.spotify.fmt:fmt-maven-plugin:check @@ -56,6 +68,13 @@ clean: package: clean mvn $(MVN_OPTS) package -DskipTests +refresh_cache: refresh_cache_branch +refresh_cache_branch: + docker inspect $(IMAGE_REPO_BRANCH) > /dev/null && docker pull $(IMAGE_REPO_BRANCH) || true +refresh_cache: refresh_cache_latest +refresh_cache_latest: + docker inspect $(IMAGE_REPO_MAIN) > /dev/null && docker pull $(IMAGE_REPO_MAIN) || true + push: docker push $(IMAGE_REPO_FINAL) @@ -63,7 +82,7 @@ push_branch: docker push $(IMAGE_REPO_BRANCH) clean_ci: - -docker rmi -f $(IMAGE_CI) $(IMAGE_REPO_BRANCH) $(IMAGE_REPO_FINAL) + -docker rmi -f $(IMAGE_CI) $(IMAGE_REPO_FINAL) -git clean -xdf . .PHONY: run package build clean test runtime-conf diff --git a/services/history-v1/Makefile b/services/history-v1/Makefile index 17159b7dfd..4344d4e546 100644 --- a/services/history-v1/Makefile +++ b/services/history-v1/Makefile @@ -6,6 +6,28 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = history-v1 BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +HERE=$(shell pwd) +export MONOREPO ?= $(shell cd ../../ && pwd) +IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME)-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/mongo-utils/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/object-persistor/package.json \ + $(MONOREPO)/libraries/overleaf-editor-core/package.json \ + $(MONOREPO)/libraries/promise-utils/package.json \ + $(MONOREPO)/libraries/redis-wrapper/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/libraries/stream-utils/package.json \ + $(MONOREPO)/services/history-v1/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ @@ -23,19 +45,17 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) clean: - -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi $(IMAGE_CI) + -docker rmi $(IMAGE_REPO_FINAL) -$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes -rm -rf reports/ -HERE=$(shell pwd) -export MONOREPO ?= $(shell cd ../../ && pwd) # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports $(IMAGE_CI) npm run --silent # Same but from the top of the monorepo RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent @@ -128,11 +148,10 @@ build: docker build \ --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --tag $(IMAGE_CI) \ + --tag $(IMAGE_CACHE) \ + --tag $(IMAGE_REPO_FINAL) \ + --cache-from $(IMAGE_CACHE) \ --file Dockerfile \ ../.. @@ -140,10 +159,10 @@ tar: $(DOCKER_COMPOSE) up tar push: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker push $(IMAGE_REPO_FINAL) push_branch: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) + docker push $(IMAGE_CACHE) .PHONY: clean \ format format_fix \ diff --git a/services/history-v1/buildscript.txt b/services/history-v1/buildscript.txt index 2e85103662..fcdb1e9865 100644 --- a/services/history-v1/buildscript.txt +++ b/services/history-v1/buildscript.txt @@ -1,6 +1,5 @@ history-v1 --dependencies=postgres,gcs,mongo,redis,s3 ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/services/notifications/Makefile b/services/notifications/Makefile index 05262acb69..d97d54de33 100644 --- a/services/notifications/Makefile +++ b/services/notifications/Makefile @@ -6,6 +6,25 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = notifications BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +HERE=$(shell pwd) +export MONOREPO ?= $(shell cd ../../ && pwd) +IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME)-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/mongo-utils/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/promise-utils/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/libraries/validation-tools/package.json \ + $(MONOREPO)/services/notifications/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ @@ -23,19 +42,17 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) clean: - -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi $(IMAGE_CI) + -docker rmi $(IMAGE_REPO_FINAL) -$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes -rm -rf reports/ -HERE=$(shell pwd) -export MONOREPO ?= $(shell cd ../../ && pwd) # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports $(IMAGE_CI) npm run --silent # Same but from the top of the monorepo RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent @@ -128,11 +145,10 @@ build: docker build \ --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --tag $(IMAGE_CI) \ + --tag $(IMAGE_CACHE) \ + --tag $(IMAGE_REPO_FINAL) \ + --cache-from $(IMAGE_CACHE) \ --file Dockerfile \ ../.. @@ -140,10 +156,10 @@ tar: $(DOCKER_COMPOSE) up tar push: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker push $(IMAGE_REPO_FINAL) push_branch: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) + docker push $(IMAGE_CACHE) .PHONY: clean \ format format_fix \ diff --git a/services/notifications/buildscript.txt b/services/notifications/buildscript.txt index dd00574061..a7a7efd6c3 100644 --- a/services/notifications/buildscript.txt +++ b/services/notifications/buildscript.txt @@ -1,6 +1,5 @@ notifications --dependencies=mongo ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/services/project-history/Makefile b/services/project-history/Makefile index 5fdaa5de99..233a057fec 100644 --- a/services/project-history/Makefile +++ b/services/project-history/Makefile @@ -6,6 +6,27 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = project-history BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +HERE=$(shell pwd) +export MONOREPO ?= $(shell cd ../../ && pwd) +IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME)-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/mongo-utils/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/overleaf-editor-core/package.json \ + $(MONOREPO)/libraries/promise-utils/package.json \ + $(MONOREPO)/libraries/redis-wrapper/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/libraries/stream-utils/package.json \ + $(MONOREPO)/services/project-history/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ @@ -23,19 +44,17 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) clean: - -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi $(IMAGE_CI) + -docker rmi $(IMAGE_REPO_FINAL) -$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes -rm -rf reports/ -HERE=$(shell pwd) -export MONOREPO ?= $(shell cd ../../ && pwd) # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports $(IMAGE_CI) npm run --silent # Same but from the top of the monorepo RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent @@ -128,11 +147,10 @@ build: docker build \ --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --tag $(IMAGE_CI) \ + --tag $(IMAGE_CACHE) \ + --tag $(IMAGE_REPO_FINAL) \ + --cache-from $(IMAGE_CACHE) \ --file Dockerfile \ ../.. @@ -140,10 +158,10 @@ tar: $(DOCKER_COMPOSE) up tar push: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker push $(IMAGE_REPO_FINAL) push_branch: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) + docker push $(IMAGE_CACHE) .PHONY: clean \ format format_fix \ diff --git a/services/project-history/buildscript.txt b/services/project-history/buildscript.txt index 1d567e09f1..826f25397f 100644 --- a/services/project-history/buildscript.txt +++ b/services/project-history/buildscript.txt @@ -1,6 +1,5 @@ project-history --dependencies=mongo,redis ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=True diff --git a/services/real-time/Makefile b/services/real-time/Makefile index 8d2f11ac8b..6fb548ef35 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -6,6 +6,23 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = real-time BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +HERE=$(shell pwd) +export MONOREPO ?= $(shell cd ../../ && pwd) +IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME)-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/redis-wrapper/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/services/real-time/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ @@ -23,19 +40,17 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) clean: - -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi $(IMAGE_CI) + -docker rmi $(IMAGE_REPO_FINAL) -$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes -rm -rf reports/ -HERE=$(shell pwd) -export MONOREPO ?= $(shell cd ../../ && pwd) # Run the linting commands in the scope of the monorepo. # Eslint and prettier (plus some configs) are on the root. RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent -RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports $(IMAGE_CI) npm run --silent # Same but from the top of the monorepo RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent @@ -128,11 +143,10 @@ build: docker build \ --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ - --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --tag $(IMAGE_CI) \ + --tag $(IMAGE_CACHE) \ + --tag $(IMAGE_REPO_FINAL) \ + --cache-from $(IMAGE_CACHE) \ --file Dockerfile \ ../.. @@ -140,10 +154,10 @@ tar: $(DOCKER_COMPOSE) up tar push: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker push $(IMAGE_REPO_FINAL) push_branch: - docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) + docker push $(IMAGE_CACHE) .PHONY: clean \ format format_fix \ diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index b16d5dde76..e2d3b40ab2 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -1,6 +1,5 @@ real-time --dependencies=redis ---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= --esmock-loader=False diff --git a/services/web/Jenkinsfile b/services/web/Jenkinsfile index 6c3947c340..7f3e784abc 100644 --- a/services/web/Jenkinsfile +++ b/services/web/Jenkinsfile @@ -58,7 +58,7 @@ pipeline { stage('Push Deps') { steps { dir('services/web') { - sh 'docker push ${AR_URL}/${IMAGE_NAME}:${BRANCH_NAME}-deps' + sh 'make push_branch' } } } diff --git a/services/web/Makefile b/services/web/Makefile index 499e19d9cf..8ef9e8a5ae 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -546,18 +546,36 @@ shellcheck_fix: IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME) -IMAGE_REPO_BRANCH ?= $(IMAGE_REPO):$(BRANCH_NAME) -IMAGE_REPO_MAIN ?= $(IMAGE_REPO):main -IMAGE_REPO_FINAL ?= $(IMAGE_REPO_BRANCH)-$(BUILD_NUMBER) +IMAGE_REPO_FINAL ?= $(IMAGE_REPO):main-$(BUILD_NUMBER) +export IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ + $(MONOREPO)/package.json \ + $(MONOREPO)/package-lock.json \ + $(MONOREPO)/libraries/access-token-encryptor/package.json \ + $(MONOREPO)/libraries/eslint-plugin/package.json \ + $(MONOREPO)/libraries/fetch-utils/package.json \ + $(MONOREPO)/libraries/logger/package.json \ + $(MONOREPO)/libraries/metrics/package.json \ + $(MONOREPO)/libraries/mongo-utils/package.json \ + $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/object-persistor/package.json \ + $(MONOREPO)/libraries/overleaf-editor-core/package.json \ + $(MONOREPO)/libraries/promise-utils/package.json \ + $(MONOREPO)/libraries/ranges-tracker/package.json \ + $(MONOREPO)/libraries/redis-wrapper/package.json \ + $(MONOREPO)/libraries/settings/package.json \ + $(MONOREPO)/libraries/stream-utils/package.json \ + $(MONOREPO)/libraries/validation-tools/package.json \ + $(MONOREPO)/services/web/package.json \ + $(MONOREPO)/patches/* \ +| sha256sum | cut -d '-' -f1) export SENTRY_RELEASE ?= ${COMMIT_SHA} build_deps: docker build --pull \ --build-arg BUILDKIT_INLINE_CACHE=1 \ - --cache-from $(IMAGE_REPO_BRANCH)-deps \ - --cache-from $(IMAGE_REPO_MAIN)-deps \ - --tag $(IMAGE_REPO_BRANCH)-deps \ + --cache-from $(IMAGE_CACHE) \ + --tag $(IMAGE_CACHE) \ --target deps \ --file Dockerfile \ ../.. @@ -578,8 +596,6 @@ build_webpack: build_webpack_once: docker build \ --build-arg SENTRY_RELEASE \ - --cache-from $(IMAGE_CI)-dev \ - --cache-from $(IMAGE_CI)-webpack \ --tag $(IMAGE_CI)-webpack \ --target webpack \ --file Dockerfile \ @@ -588,7 +604,6 @@ build_webpack_once: build_pug: docker build \ --build-arg SENTRY_RELEASE \ - --cache-from $(IMAGE_CI)-dev \ --tag $(IMAGE_CI)-pug \ --target pug \ --file Dockerfile \ @@ -597,9 +612,6 @@ build_pug: build: docker build \ --build-arg SENTRY_RELEASE \ - --cache-from $(IMAGE_CI)-webpack \ - --cache-from $(IMAGE_CI)-pug \ - --cache-from $(IMAGE_REPO_FINAL) \ --tag $(IMAGE_REPO_FINAL) \ --target app \ --file Dockerfile \ @@ -608,6 +620,9 @@ build: publish: docker push $(IMAGE_REPO_FINAL) +push_branch: + docker push $(IMAGE_CACHE) + SENTRY_IMAGE=getsentry/sentry-cli:2.16.1 sentry_prefetch: docker pull $(SENTRY_IMAGE)