[monorepo] move retries from the outside to the per-test level (#31528)

* [monorepo] move retries from the outside to the per-test level

* [web] increase hookTimeout in CI

* [monorepo] consolidate test retries

- do not retry unit tests
- only retry in ci, optionally locally with "RETRIES=3 make test..."
- add retries for web

Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com>

---------

Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com>
GitOrigin-RevId: 86e45edcfb087d18e0e957ad6df9a6105dcd5770
This commit is contained in:
Jakob Ackermann
2026-02-16 09:06:48 +00:00
committed by Copybot
co-authored by Eric Mc Sween
parent 1f91ca473d
commit dbf6401f22
45 changed files with 45 additions and 11 deletions
+1
View File
@@ -9,6 +9,7 @@ PROJECT_NAME = chat
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
HERE=$(shell pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
export RETRIES ?= $(if $(CI),3,0)
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
+1
View File
@@ -25,6 +25,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
RETRIES:
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
+1
View File
@@ -35,6 +35,7 @@ services:
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/chat
environment:
RETRIES:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
+1 -1
View File
@@ -9,7 +9,7 @@
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit --retries=$RETRIES $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
"lint": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --max-warnings 0 --format unix .",
"lint:fix": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --fix .",
+1
View File
@@ -9,6 +9,7 @@ PROJECT_NAME = clsi
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
HERE=$(shell pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
export RETRIES ?= $(if $(CI),3,0)
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
+1
View File
@@ -21,6 +21,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
RETRIES:
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
+1
View File
@@ -34,6 +34,7 @@ services:
- /var/run/docker.sock:/var/run/docker.sock
working_dir: /overleaf/services/clsi
environment:
RETRIES:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
+1 -1
View File
@@ -6,7 +6,7 @@
"type": "module",
"scripts": {
"start": "node app.js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit --retries=$RETRIES $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "vitest --config ./vitest.config.unit.cjs",
"test:unit": "npm run test:unit:_run",
+1
View File
@@ -9,6 +9,7 @@ PROJECT_NAME = contacts
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
HERE=$(shell pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
export RETRIES ?= $(if $(CI),3,0)
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
+1
View File
@@ -25,6 +25,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
RETRIES:
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
+1
View File
@@ -35,6 +35,7 @@ services:
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/contacts
environment:
RETRIES:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
+1 -1
View File
@@ -6,7 +6,7 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
"test:acceptance:_run": "mocha --loader=esmock --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --loader=esmock --recursive --timeout 15000 --exit --retries=$RETRIES $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --loader=esmock --recursive --exit $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
+1
View File
@@ -9,6 +9,7 @@ PROJECT_NAME = docstore
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
HERE=$(shell pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
export RETRIES ?= $(if $(CI),3,0)
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
+1
View File
@@ -27,6 +27,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
RETRIES:
MONGO_HOST: mongo
POSTGRES_HOST: postgres
GCS_API_ENDPOINT: http://gcs:9090
+1
View File
@@ -36,6 +36,7 @@ services:
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/docstore
environment:
RETRIES:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
GCS_API_ENDPOINT: http://gcs:9090
+1 -1
View File
@@ -6,7 +6,7 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit --retries=$RETRIES $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "vitest --config ./vitest.config.unit.cjs",
"test:unit": "npm run test:unit:_run",
+1
View File
@@ -9,6 +9,7 @@ PROJECT_NAME = document-updater
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
HERE=$(shell pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
export RETRIES ?= $(if $(CI),3,0)
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
@@ -32,6 +32,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
RETRIES:
REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
HISTORY_REDIS_HOST: redis_test
@@ -42,6 +42,7 @@ services:
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/document-updater
environment:
RETRIES:
REDIS_HOST: redis_test
HISTORY_REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
+1 -1
View File
@@ -5,7 +5,7 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit --retries=$RETRIES $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
+1
View File
@@ -9,6 +9,7 @@ PROJECT_NAME = filestore
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
HERE=$(shell pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
export RETRIES ?= $(if $(CI),3,0)
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
+1
View File
@@ -25,6 +25,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
RETRIES:
MONGO_HOST: mongo
POSTGRES_HOST: postgres
AWS_S3_ENDPOINT: https://minio:9000
+1
View File
@@ -38,6 +38,7 @@ services:
- minio-certs:/certs
working_dir: /overleaf/services/filestore
environment:
RETRIES:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
AWS_S3_ENDPOINT: https://minio:9000
+1 -1
View File
@@ -12,7 +12,7 @@
"start": "node app.js",
"nodemon": "node --watch app.js",
"lint": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --max-warnings 0 --format unix .",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit --retries=$RETRIES $@ test/acceptance/js",
"test:unit:_run": "vitest --config ./vitest.config.unit.cjs",
"lint:fix": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --fix .",
"types:check": "tsc --noEmit"
+1
View File
@@ -9,6 +9,7 @@ PROJECT_NAME = history-v1
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
HERE=$(shell pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
export RETRIES ?= $(if $(CI),3,0)
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
@@ -35,6 +35,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
RETRIES:
REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
HISTORY_REDIS_HOST: redis_test
+1
View File
@@ -52,6 +52,7 @@ services:
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/history-v1
environment:
RETRIES:
REDIS_HOST: redis_test
HISTORY_REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
+1 -1
View File
@@ -67,7 +67,7 @@
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit --retries=$RETRIES $@ test/acceptance/js",
"nodemon": "node --watch app.js",
"migrate": "knex migrate:latest",
"delete_old_chunks": "node storage/tasks/delete_old_chunks.js",
+1
View File
@@ -9,6 +9,7 @@ PROJECT_NAME = notifications
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
HERE=$(shell pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
export RETRIES ?= $(if $(CI),3,0)
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
@@ -27,6 +27,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
RETRIES:
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
@@ -37,6 +37,7 @@ services:
- ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
working_dir: /overleaf/services/notifications
environment:
RETRIES:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
+1
View File
@@ -9,6 +9,7 @@ PROJECT_NAME = project-history
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
HERE=$(shell pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
export RETRIES ?= $(if $(CI),3,0)
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
@@ -32,6 +32,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
RETRIES:
REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
HISTORY_REDIS_HOST: redis_test
@@ -42,6 +42,7 @@ services:
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/project-history
environment:
RETRIES:
REDIS_HOST: redis_test
HISTORY_REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
+1 -1
View File
@@ -9,7 +9,7 @@
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"start": "node app.js",
"nodemon": "node --watch app.js",
"test:acceptance:_run": "mocha --loader=esmock --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --loader=esmock --recursive --timeout 15000 --exit --retries=$RETRIES $@ test/acceptance/js",
"test:unit:_run": "mocha --loader=esmock --recursive --exit $@ test/unit/js",
"lint": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --max-warnings 0 --format unix .",
"lint:fix": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --fix .",
+1
View File
@@ -9,6 +9,7 @@ PROJECT_NAME = real-time
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
HERE=$(shell pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
export RETRIES ?= $(if $(CI),3,0)
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
+1
View File
@@ -30,6 +30,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
RETRIES:
REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
HISTORY_REDIS_HOST: redis_test
+1
View File
@@ -36,6 +36,7 @@ services:
- ../../libraries:/overleaf/libraries
working_dir: /overleaf/services/real-time
environment:
RETRIES:
REDIS_HOST: redis_test
HISTORY_REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
+1 -1
View File
@@ -6,7 +6,7 @@
"type": "module",
"scripts": {
"start": "node app.js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit --retries=$RETRIES $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "vitest --config ./vitest.config.unit.cjs",
"test:unit": "npm run test:unit:_run",
+1
View File
@@ -1,4 +1,5 @@
DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
export RETRIES ?= $(if $(CI),3,0)
BUILD_NUMBER ?= local
ifeq ($(BRANCH_NAME),)
+1
View File
@@ -2,6 +2,7 @@ BUILD_DIR_NAME ?= web
export MODULE_NAME := $(notdir $(shell pwd))
MODULE_DIR := modules/$(MODULE_NAME)
PROJECT_NAME = web
export RETRIES ?= $(if $(CI),3,0)
export OVERLEAF_CONFIG = /overleaf/services/web/$(MODULE_DIR)/test/acceptance/config/settings.test.js
export BASE_CONFIG ?= /overleaf/services/web/test/acceptance/config/settings.test.saas.js
+1 -1
View File
@@ -9,4 +9,4 @@ if [[ "$COVERAGE_ACCEPTANCE_TESTS" == "true" ]]; then
mocha_prefix+=("c8" "--reporter=clover" "--all" "--include={app.mjs,app/**/*.{mjs,js},modules/*/index.mjs,modules/*/app/**/*.{mjs,js}}" "--exclude=app/src/Features/Metadata/packageMapping.mjs" "--report-dir=${dir}")
fi
"${mocha_prefix[@]}" mocha --recursive --timeout 25000 --grep="$MOCHA_GREP" --require test/acceptance/bootstrap.js "$@"
"${mocha_prefix[@]}" mocha --recursive --timeout 25000 --grep="$MOCHA_GREP" --require test/acceptance/bootstrap.js --retries="$RETRIES" "$@"
+1
View File
@@ -52,6 +52,7 @@ services:
REDIS_HOST: redis_test
COVERAGE_ACCEPTANCE_TESTS:
MONOREPO:
RETRIES:
extra_hosts:
- "www.overleaf.test:127.0.0.1"
volumes:
+1
View File
@@ -56,6 +56,7 @@ services:
MONGO_SOCKET_TIMEOUT: 300000
REDIS_HOST: redis_test
MONOREPO:
RETRIES:
# OVERLEAF_ALLOW_ANONYMOUS_READ_AND_WRITE_SHARING: 'true'
extra_hosts:
+1 -1
View File
@@ -57,7 +57,7 @@ module.exports = defineConfig({
},
],
...reporterOptions,
hookTimeout: COVERAGE_ENABLED ? 20_000 : 10_000,
hookTimeout: process.env.CI ? 20_000 : 10_000,
coverage: {
enabled: COVERAGE_ENABLED,
// Add 'sequential' / 'parallel' to the folder