agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-14 14:13:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

[monorepo] migrate build scripts to Jenkins (#29019)

Browse Source 
* [monorepo] migrate build scripts to Jenkins

* [fraud-dash] fix integration with monorepo

GitOrigin-RevId: 6a30ec5ea68a1eae9b663faf9ccdc741ec7f9cf8
This commit is contained in:
Jakob Ackermann
2025-10-13 08:07:11 +00:00
committed by Copybot
parent 9813bc4b51
commit fd0d21fbbd
142 changed files with 3298 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/chat/.mocharc.cjs
+13
View File
@@ -0,0 +1,13 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: '/overleaf/node_modules/mocha-multi-reporters',
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
services/chat/.mocharc.json
-3
View File
@@ -1,3 +0,0 @@
{
"require": "test/setup.js"
}
services/chat/Dockerfile
+13 -3
View File
@@ -13,12 +13,22 @@ RUN mkdir /home/node/.config && chown node:node /home/node/.config
FROM base AS app
COPY package.json package-lock.json /overleaf/
COPY services/chat/package.json /overleaf/services/chat/
COPY libraries/ /overleaf/libraries/
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY services/chat/package.json /overleaf/services/chat/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY services/chat/ /overleaf/services/chat/
FROM app
services/chat/Jenkinsfile
Vendored
+125
View File
@@ -0,0 +1,125 @@
// Autogenerated by build scripts. Do not edit.
pipeline {
agent {
node {
label 'jenkins-agent-web'
customWorkspace '/workspace'
}
}
options {
timestamps()
timeout(time: 15, unit: 'MINUTES')
}
environment {
BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}"
JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}"
BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}"
DOCKER_COMPOSE_FLAGS = '-f docker-compose.ci.yml'
COMMIT_SHA = "${GIT_COMMIT}"
SHORT_SHA = "${GIT_COMMIT.take(7)}"
}
stages {
stage('Stage 1') {
parallel {
stage('Build') {
steps {
dir('services/chat') {
retry(count: 3) {
sh 'make build'
}
}
}
}
stage('Create reports folder') {
steps {
sh 'mkdir services/chat/reports'
}
}
}
}
stage('Stage 2') {
parallel {
stage('Push Branch Image') {
steps {
dir('services/chat') {
sh 'make push_branch'
}
}
}
stage('Shellcheck') {
steps {
dir('services/chat') {
sh 'make shellcheck'
}
}
}
stage('Lint') {
steps {
dir('services/chat') {
sh 'make lint_ci'
}
}
post {
always {
recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'chat-eslint', name: 'chat eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'services/chat/reports/eslint.json')]
}
}
}
stage('Format') {
steps {
dir('services/chat') {
sh 'make format_ci'
}
}
}
stage('Typecheck') {
steps {
dir('services/chat') {
sh 'make typecheck_ci'
}
}
}
stage('Test Unit') {
steps {
dir('services/chat') {
retry(count: 3) {
sh 'make test_unit'
}
}
}
}
stage('Test Acceptance') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance"
}
steps {
dir('services/chat') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
}
}
stage('Push Production') {
steps {
dir('services/chat') {
sh 'make push'
}
}
}
}
post {
always {
junit checksName: 'chat test results', testResults: 'services/chat/reports/junit-*.xml'
}
cleanup {
dir('services/chat') {
sh 'make clean'
}
sh 'make clean_jenkins -j10'
}
}
}
services/chat/Makefile
+9 -6
View File
@@ -27,14 +27,15 @@ clean:
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
-rm -rf reports/
HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent
@@ -70,7 +71,8 @@ lint:
$(RUN_LINTING) lint
lint_ci:
$(RUN_LINTING_CI) lint
-$(RUN_LINTING_CI) lint -- --format json --output-file reports/eslint.json
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
lint_fix:
$(RUN_LINTING) lint:fix
@@ -137,10 +139,11 @@ build:
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push_branch:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)
.PHONY: clean \
format format_fix \
services/chat/docker-compose.ci.yml
+4
View File
@@ -7,10 +7,12 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
volumes:
- ./reports:/overleaf/services/chat/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 --
command: npm run test:unit:_run
environment:
CI:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
@@ -23,6 +25,7 @@ services:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
@@ -30,6 +33,7 @@ services:
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
volumes:
- ./reports:/overleaf/services/chat/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
services/chat/package.json
+4 -2
View File
@@ -9,8 +9,8 @@
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
"lint": "eslint --max-warnings 0 --format unix .",
"format": "prettier --list-different $PWD/'**/{*.*js,*.ts}'",
"format:fix": "prettier --write $PWD/'**/{*.*js,*.ts}'",
@@ -34,6 +34,8 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"request": "^2.88.2",
"sandboxed-module": "^2.0.4",
"sinon": "^9.2.4",
services/chat/test/mocha-multi-reporters.cjs
+8
View File
@@ -0,0 +1,8 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
},
}
services/clsi/.mocharc.cjs
+13
View File
@@ -0,0 +1,13 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: '/overleaf/node_modules/mocha-multi-reporters',
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
services/clsi/.mocharc.json
-3
View File
@@ -1,3 +0,0 @@
{
"require": "test/setup.js"
}
services/clsi/Dockerfile
+15 -3
View File
@@ -17,12 +17,24 @@ RUN mkdir /home/node/.config && chown node:node /home/node/.config
FROM base AS app
COPY package.json package-lock.json /overleaf/
COPY services/clsi/package.json /overleaf/services/clsi/
COPY libraries/ /overleaf/libraries/
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
COPY services/clsi/package.json /overleaf/services/clsi/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY services/clsi/ /overleaf/services/clsi/
FROM app
services/clsi/Jenkinsfile
Vendored
+125
View File
@@ -0,0 +1,125 @@
// Autogenerated by build scripts. Do not edit.
pipeline {
agent {
node {
label 'jenkins-agent-web'
customWorkspace '/workspace'
}
}
options {
timestamps()
timeout(time: 15, unit: 'MINUTES')
}
environment {
BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}"
JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}"
BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}"
DOCKER_COMPOSE_FLAGS = '-f docker-compose.ci.yml'
COMMIT_SHA = "${GIT_COMMIT}"
SHORT_SHA = "${GIT_COMMIT.take(7)}"
}
stages {
stage('Stage 1') {
parallel {
stage('Build') {
steps {
dir('services/clsi') {
retry(count: 3) {
sh 'make build'
}
}
}
}
stage('Create reports folder') {
steps {
sh 'mkdir services/clsi/reports'
}
}
}
}
stage('Stage 2') {
parallel {
stage('Push Branch Image') {
steps {
dir('services/clsi') {
sh 'make push_branch'
}
}
}
stage('Shellcheck') {
steps {
dir('services/clsi') {
sh 'make shellcheck'
}
}
}
stage('Lint') {
steps {
dir('services/clsi') {
sh 'make lint_ci'
}
}
post {
always {
recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'clsi-eslint', name: 'clsi eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'services/clsi/reports/eslint.json')]
}
}
}
stage('Format') {
steps {
dir('services/clsi') {
sh 'make format_ci'
}
}
}
stage('Typecheck') {
steps {
dir('services/clsi') {
sh 'make typecheck_ci'
}
}
}
stage('Test Unit') {
steps {
dir('services/clsi') {
retry(count: 3) {
sh 'make test_unit'
}
}
}
}
stage('Test Acceptance') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance"
}
steps {
dir('services/clsi') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
}
}
stage('Push Production') {
steps {
dir('services/clsi') {
sh 'make push'
}
}
}
}
post {
always {
junit checksName: 'clsi test results', testResults: 'services/clsi/reports/junit-*.xml'
}
cleanup {
dir('services/clsi') {
sh 'make clean'
}
sh 'make clean_jenkins -j10'
}
}
}
services/clsi/Makefile
+9 -6
View File
@@ -27,15 +27,16 @@ clean:
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
-rm -rf reports/
-git clean -dfX cache compiles output
HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent
@@ -71,7 +72,8 @@ lint:
$(RUN_LINTING) lint
lint_ci:
$(RUN_LINTING_CI) lint
-$(RUN_LINTING_CI) lint -- --format json --output-file reports/eslint.json
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
lint_fix:
$(RUN_LINTING) lint:fix
@@ -138,10 +140,11 @@ build:
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push_branch:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)
.PHONY: clean \
format format_fix \
services/clsi/bin/acceptance_test
+1 -1
View File
@@ -1,4 +1,4 @@
#!/bin/bash
set -e;
MOCHA="node_modules/.bin/mocha --recursive --reporter spec --timeout 15000"
MOCHA="node_modules/.bin/mocha --recursive --timeout 15000"
$MOCHA "$@"
services/clsi/docker-compose.ci.yml
+3
View File
@@ -7,6 +7,7 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
command: npm run test:unit:_run
environment:
CI:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
@@ -16,6 +17,7 @@ services:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
@@ -32,6 +34,7 @@ services:
SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
volumes:
- ./reports:/overleaf/services/clsi/reports
- ./compiles:/overleaf/services/clsi/compiles
- /var/run/docker.sock:/var/run/docker.sock
command: npm run test:acceptance
services/clsi/package.json
+4 -2
View File
@@ -5,9 +5,9 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
@@ -41,6 +41,8 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"mock-fs": "^5.1.2",
"node-fetch": "^2.7.0",
"sandboxed-module": "^2.0.4",
services/clsi/test/mocha-multi-reporters.cjs
+8
View File
@@ -0,0 +1,8 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
},
}
services/contacts/.mocharc.cjs
+13
View File
@@ -0,0 +1,13 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: '/overleaf/node_modules/mocha-multi-reporters',
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
services/contacts/.mocharc.json
-3
View File
@@ -1,3 +0,0 @@
{
"require": "test/setup.js"
}
services/contacts/Dockerfile
+13 -3
View File
@@ -13,12 +13,22 @@ RUN mkdir /home/node/.config && chown node:node /home/node/.config
FROM base AS app
COPY package.json package-lock.json /overleaf/
COPY services/contacts/package.json /overleaf/services/contacts/
COPY libraries/ /overleaf/libraries/
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY services/contacts/package.json /overleaf/services/contacts/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY services/contacts/ /overleaf/services/contacts/
FROM app
services/contacts/Jenkinsfile
Vendored
+54 -33
View File
@@ -1,3 +1,5 @@
// Autogenerated by build scripts. Do not edit.
pipeline {
agent {
node {
@@ -7,16 +9,9 @@ pipeline {
}
options {
timestamps()
parallelsAlwaysFailFast()
retry(3)
timeout(time: 15, unit: 'MINUTES')
}
environment {
IMAGE_NAME = 'contacts'
PROJECT_NAME = 'contacts'
IMAGE_REPO = 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/contacts'
AR_REPO_LOCATION = 'us-east1'
AR_URL = 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker'
BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}"
JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}"
BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}"
@@ -25,61 +20,84 @@ pipeline {
SHORT_SHA = "${GIT_COMMIT.take(7)}"
}
stages {
stage ('Build') {
steps {
dir ('services/contacts') {
sh 'make build'
}
}
}
stage ('Tests') {
stage('Stage 1') {
parallel {
stage ('Push Branch Image') {
stage('Build') {
steps {
dir ('services/contacts') {
sh 'docker push ${AR_URL}/${IMAGE_NAME}:${BRANCH_NAME}'
dir('services/contacts') {
retry(count: 3) {
sh 'make build'
}
}
}
}
stage ('Shellcheck') {
stage('Create reports folder') {
steps {
dir ('services/contacts') {
sh 'mkdir services/contacts/reports'
}
}
}
}
stage('Stage 2') {
parallel {
stage('Push Branch Image') {
steps {
dir('services/contacts') {
sh 'make push_branch'
}
}
}
stage('Shellcheck') {
steps {
dir('services/contacts') {
sh 'make shellcheck'
}
}
}
stage ('Lint') {
stage('Lint') {
steps {
dir ('services/contacts') {
dir('services/contacts') {
sh 'make lint_ci'
}
}
post {
always {
recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'contacts-eslint', name: 'contacts eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'services/contacts/reports/eslint.json')]
}
}
}
stage ('Format') {
stage('Format') {
steps {
dir ('services/contacts') {
dir('services/contacts') {
sh 'make format_ci'
}
}
}
stage ('Typecheck') {
stage('Typecheck') {
steps {
dir ('services/contacts') {
dir('services/contacts') {
sh 'make typecheck_ci'
}
}
}
stage ('Test Unit') {
stage('Test Unit') {
steps {
dir ('services/contacts') {
sh 'make test_unit'
dir('services/contacts') {
retry(count: 3) {
sh 'make test_unit'
}
}
}
}
stage ('Test Acceptance') {
stage('Test Acceptance') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance"
}
steps {
dir ('services/contacts') {
sh 'make test_acceptance'
dir('services/contacts') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
@@ -88,12 +106,15 @@ pipeline {
stage('Push Production') {
steps {
dir('services/contacts') {
sh 'docker push ${AR_URL}/${IMAGE_NAME}:${BRANCH_NAME}-${BUILD_NUMBER}'
sh 'make push'
}
}
}
}
post {
always {
junit checksName: 'contacts test results', testResults: 'services/contacts/reports/junit-*.xml'
}
cleanup {
dir('services/contacts') {
sh 'make clean'
services/contacts/Makefile
+9 -6
View File
@@ -27,14 +27,15 @@ clean:
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
-rm -rf reports/
HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/contacts/reports:/overleaf/services/contacts/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent
@@ -70,7 +71,8 @@ lint:
$(RUN_LINTING) lint
lint_ci:
$(RUN_LINTING_CI) lint
-$(RUN_LINTING_CI) lint -- --format json --output-file reports/eslint.json
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
lint_fix:
$(RUN_LINTING) lint:fix
@@ -137,10 +139,11 @@ build:
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push_branch:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)
.PHONY: clean \
format format_fix \
services/contacts/docker-compose.ci.yml
+4
View File
@@ -7,10 +7,12 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
volumes:
- ./reports:/overleaf/services/contacts/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 --
command: npm run test:unit:_run
environment:
CI:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
@@ -23,6 +25,7 @@ services:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
@@ -30,6 +33,7 @@ services:
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
volumes:
- ./reports:/overleaf/services/contacts/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
services/contacts/package.json
+4 -2
View File
@@ -6,9 +6,9 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
"test:acceptance:_run": "mocha --loader=esmock --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --loader=esmock --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --loader=esmock --recursive --reporter spec --exit $@ test/unit/js",
"test:unit:_run": "mocha --loader=esmock --recursive --exit $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
@@ -35,6 +35,8 @@
"chai-as-promised": "^7.1.1",
"esmock": "^2.6.3",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"sinon": "~9.0.1",
"sinon-chai": "^3.7.0",
"typescript": "^5.0.4"
services/contacts/test/mocha-multi-reporters.cjs
+8
View File
@@ -0,0 +1,8 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
},
}
services/docstore/.mocharc.cjs
+13
View File
@@ -0,0 +1,13 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: '/overleaf/node_modules/mocha-multi-reporters',
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
services/docstore/.mocharc.json
-3
View File
@@ -1,3 +0,0 @@
{
"require": "test/setup.js"
}
services/docstore/Dockerfile
+19 -3
View File
@@ -13,12 +13,28 @@ RUN mkdir /home/node/.config && chown node:node /home/node/.config
FROM base AS app
COPY package.json package-lock.json /overleaf/
COPY services/docstore/package.json /overleaf/services/docstore/
COPY libraries/ /overleaf/libraries/
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/object-persistor/package.json /overleaf/libraries/object-persistor/package.json
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
COPY services/docstore/package.json /overleaf/services/docstore/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/object-persistor/ /overleaf/libraries/object-persistor/
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY services/docstore/ /overleaf/services/docstore/
FROM app
services/docstore/Jenkinsfile
Vendored
+125
View File
@@ -0,0 +1,125 @@
// Autogenerated by build scripts. Do not edit.
pipeline {
agent {
node {
label 'jenkins-agent-web'
customWorkspace '/workspace'
}
}
options {
timestamps()
timeout(time: 15, unit: 'MINUTES')
}
environment {
BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}"
JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}"
BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}"
DOCKER_COMPOSE_FLAGS = '-f docker-compose.ci.yml'
COMMIT_SHA = "${GIT_COMMIT}"
SHORT_SHA = "${GIT_COMMIT.take(7)}"
}
stages {
stage('Stage 1') {
parallel {
stage('Build') {
steps {
dir('services/docstore') {
retry(count: 3) {
sh 'make build'
}
}
}
}
stage('Create reports folder') {
steps {
sh 'mkdir services/docstore/reports'
}
}
}
}
stage('Stage 2') {
parallel {
stage('Push Branch Image') {
steps {
dir('services/docstore') {
sh 'make push_branch'
}
}
}
stage('Shellcheck') {
steps {
dir('services/docstore') {
sh 'make shellcheck'
}
}
}
stage('Lint') {
steps {
dir('services/docstore') {
sh 'make lint_ci'
}
}
post {
always {
recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'docstore-eslint', name: 'docstore eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'services/docstore/reports/eslint.json')]
}
}
}
stage('Format') {
steps {
dir('services/docstore') {
sh 'make format_ci'
}
}
}
stage('Typecheck') {
steps {
dir('services/docstore') {
sh 'make typecheck_ci'
}
}
}
stage('Test Unit') {
steps {
dir('services/docstore') {
retry(count: 3) {
sh 'make test_unit'
}
}
}
}
stage('Test Acceptance') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance"
}
steps {
dir('services/docstore') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
}
}
stage('Push Production') {
steps {
dir('services/docstore') {
sh 'make push'
}
}
}
}
post {
always {
junit checksName: 'docstore test results', testResults: 'services/docstore/reports/junit-*.xml'
}
cleanup {
dir('services/docstore') {
sh 'make clean'
}
sh 'make clean_jenkins -j10'
}
}
}
services/docstore/Makefile
+9 -6
View File
@@ -27,14 +27,15 @@ clean:
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
-rm -rf reports/
HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent
@@ -70,7 +71,8 @@ lint:
$(RUN_LINTING) lint
lint_ci:
$(RUN_LINTING_CI) lint
-$(RUN_LINTING_CI) lint -- --format json --output-file reports/eslint.json
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
lint_fix:
$(RUN_LINTING) lint:fix
@@ -137,10 +139,11 @@ build:
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push_branch:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)
.PHONY: clean \
format format_fix \
services/docstore/docker-compose.ci.yml
+4
View File
@@ -7,10 +7,12 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
volumes:
- ./reports:/overleaf/services/docstore/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 --
command: npm run test:unit:_run
environment:
CI:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
@@ -23,6 +25,7 @@ services:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
@@ -33,6 +36,7 @@ services:
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
volumes:
- ./reports:/overleaf/services/docstore/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
services/docstore/package.json
+4 -2
View File
@@ -5,9 +5,9 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
@@ -40,6 +40,8 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"sandboxed-module": "~2.0.4",
"sinon": "~9.0.2",
"sinon-chai": "^3.7.0",
services/docstore/test/mocha-multi-reporters.cjs
+8
View File
@@ -0,0 +1,8 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
},
}
services/document-updater/.mocharc.cjs
+13
View File
@@ -0,0 +1,13 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: '/overleaf/node_modules/mocha-multi-reporters',
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
services/document-updater/.mocharc.json
-3
View File
@@ -1,3 +0,0 @@
{
"require": "test/setup.js"
}
services/document-updater/Dockerfile
+21 -3
View File
@@ -13,12 +13,30 @@ RUN mkdir /home/node/.config && chown node:node /home/node/.config
FROM base AS app
COPY package.json package-lock.json /overleaf/
COPY services/document-updater/package.json /overleaf/services/document-updater/
COPY libraries/ /overleaf/libraries/
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/overleaf-editor-core/package.json /overleaf/libraries/overleaf-editor-core/package.json
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
COPY libraries/ranges-tracker/package.json /overleaf/libraries/ranges-tracker/package.json
COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY services/document-updater/package.json /overleaf/services/document-updater/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/overleaf-editor-core/ /overleaf/libraries/overleaf-editor-core/
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/ranges-tracker/ /overleaf/libraries/ranges-tracker/
COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY services/document-updater/ /overleaf/services/document-updater/
FROM app
services/document-updater/Jenkinsfile
Vendored
+125
View File
@@ -0,0 +1,125 @@
// Autogenerated by build scripts. Do not edit.
pipeline {
agent {
node {
label 'jenkins-agent-web'
customWorkspace '/workspace'
}
}
options {
timestamps()
timeout(time: 15, unit: 'MINUTES')
}
environment {
BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}"
JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}"
BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}"
DOCKER_COMPOSE_FLAGS = '-f docker-compose.ci.yml'
COMMIT_SHA = "${GIT_COMMIT}"
SHORT_SHA = "${GIT_COMMIT.take(7)}"
}
stages {
stage('Stage 1') {
parallel {
stage('Build') {
steps {
dir('services/document-updater') {
retry(count: 3) {
sh 'make build'
}
}
}
}
stage('Create reports folder') {
steps {
sh 'mkdir services/document-updater/reports'
}
}
}
}
stage('Stage 2') {
parallel {
stage('Push Branch Image') {
steps {
dir('services/document-updater') {
sh 'make push_branch'
}
}
}
stage('Shellcheck') {
steps {
dir('services/document-updater') {
sh 'make shellcheck'
}
}
}
stage('Lint') {
steps {
dir('services/document-updater') {
sh 'make lint_ci'
}
}
post {
always {
recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'document-updater-eslint', name: 'document-updater eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'services/document-updater/reports/eslint.json')]
}
}
}
stage('Format') {
steps {
dir('services/document-updater') {
sh 'make format_ci'
}
}
}
stage('Typecheck') {
steps {
dir('services/document-updater') {
sh 'make typecheck_ci'
}
}
}
stage('Test Unit') {
steps {
dir('services/document-updater') {
retry(count: 3) {
sh 'make test_unit'
}
}
}
}
stage('Test Acceptance') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance"
}
steps {
dir('services/document-updater') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
}
}
stage('Push Production') {
steps {
dir('services/document-updater') {
sh 'make push'
}
}
}
}
post {
always {
junit checksName: 'document-updater test results', testResults: 'services/document-updater/reports/junit-*.xml'
}
cleanup {
dir('services/document-updater') {
sh 'make clean'
}
sh 'make clean_jenkins -j10'
}
}
}
services/document-updater/Makefile
+9 -6
View File
@@ -27,14 +27,15 @@ clean:
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
-rm -rf reports/
HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent
@@ -70,7 +71,8 @@ lint:
$(RUN_LINTING) lint
lint_ci:
$(RUN_LINTING_CI) lint
-$(RUN_LINTING_CI) lint -- --format json --output-file reports/eslint.json
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
lint_fix:
$(RUN_LINTING) lint:fix
@@ -137,10 +139,11 @@ build:
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push_branch:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)
.PHONY: clean \
format format_fix \
services/document-updater/docker-compose.ci.yml
+4
View File
@@ -7,10 +7,12 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
volumes:
- ./reports:/overleaf/services/document-updater/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 --
command: npm run test:unit:_run
environment:
CI:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
@@ -29,6 +31,7 @@ services:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
@@ -40,6 +43,7 @@ services:
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
volumes:
- ./reports:/overleaf/services/document-updater/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
services/document-updater/package.json
+4 -2
View File
@@ -5,9 +5,9 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"benchmark:apply": "node benchmarks/apply",
@@ -45,6 +45,8 @@
"chai-as-promised": "^7.1.1",
"cluster-key-slot": "^1.0.5",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"sandboxed-module": "^2.0.4",
"sinon": "^9.2.4",
"sinon-chai": "^3.7.0",
services/document-updater/test/mocha-multi-reporters.cjs
+8
View File
@@ -0,0 +1,8 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
},
}
services/filestore/.mocharc.cjs
+13
View File
@@ -0,0 +1,13 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: '/overleaf/node_modules/mocha-multi-reporters',
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
services/filestore/.mocharc.json
-3
View File
@@ -1,3 +0,0 @@
{
"require": "test/setup.js"
}
services/filestore/Dockerfile
+15 -3
View File
@@ -15,12 +15,24 @@ RUN mkdir /home/node/.config && chown node:node /home/node/.config
FROM base AS app
COPY package.json package-lock.json /overleaf/
COPY services/filestore/package.json /overleaf/services/filestore/
COPY libraries/ /overleaf/libraries/
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/object-persistor/package.json /overleaf/libraries/object-persistor/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
COPY services/filestore/package.json /overleaf/services/filestore/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/object-persistor/ /overleaf/libraries/object-persistor/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY services/filestore/ /overleaf/services/filestore/
FROM app
services/filestore/Jenkinsfile
Vendored
+152
View File
@@ -0,0 +1,152 @@
// Autogenerated by build scripts. Do not edit.
pipeline {
agent {
node {
label 'jenkins-agent-web'
customWorkspace '/workspace'
}
}
options {
timestamps()
timeout(time: 15, unit: 'MINUTES')
}
environment {
BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}"
JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}"
BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}"
DOCKER_COMPOSE_FLAGS = '-f docker-compose.ci.yml'
COMMIT_SHA = "${GIT_COMMIT}"
SHORT_SHA = "${GIT_COMMIT.take(7)}"
}
stages {
stage('Stage 1') {
parallel {
stage('Build') {
steps {
dir('services/filestore') {
retry(count: 3) {
sh 'make build'
}
}
}
}
stage('Create reports folder') {
steps {
sh 'mkdir services/filestore/reports'
}
}
}
}
stage('Stage 2') {
parallel {
stage('Push Branch Image') {
steps {
dir('services/filestore') {
sh 'make push_branch'
}
}
}
stage('Shellcheck') {
steps {
dir('services/filestore') {
sh 'make shellcheck'
}
}
}
stage('Lint') {
steps {
dir('services/filestore') {
sh 'make lint_ci'
}
}
post {
always {
recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'filestore-eslint', name: 'filestore eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'services/filestore/reports/eslint.json')]
}
}
}
stage('Format') {
steps {
dir('services/filestore') {
sh 'make format_ci'
}
}
}
stage('Typecheck') {
steps {
dir('services/filestore') {
sh 'make typecheck_ci'
}
}
}
stage('Test Unit') {
steps {
dir('services/filestore') {
retry(count: 3) {
sh 'make test_unit'
}
}
}
}
stage('Test Acceptance SHARD_01_') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance_shard_01_x"
MOCHA_GREP = "SHARD_01_"
}
steps {
dir('services/filestore') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
stage('Test Acceptance SHARD_02_') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance_shard_02_x"
MOCHA_GREP = "SHARD_02_"
}
steps {
dir('services/filestore') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
stage('Test Acceptance SHARD_03_') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance_shard_03_x"
MOCHA_GREP = "SHARD_03_"
}
steps {
dir('services/filestore') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
}
}
stage('Push Production') {
steps {
dir('services/filestore') {
sh 'make push'
}
}
}
}
post {
always {
junit checksName: 'filestore test results', testResults: 'services/filestore/reports/junit-*.xml'
}
cleanup {
dir('services/filestore') {
sh 'make clean'
}
sh 'make clean_jenkins -j10'
}
}
}
services/filestore/Makefile
+9 -6
View File
@@ -27,15 +27,16 @@ clean:
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
-rm -rf reports/
-git clean -dfX uploads template_files
HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent
@@ -71,7 +72,8 @@ lint:
$(RUN_LINTING) lint
lint_ci:
$(RUN_LINTING_CI) lint
-$(RUN_LINTING_CI) lint -- --format json --output-file reports/eslint.json
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
lint_fix:
$(RUN_LINTING) lint:fix
@@ -138,10 +140,11 @@ build:
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push_branch:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)
.PHONY: clean \
format format_fix \
services/filestore/docker-compose.ci.yml
+3
View File
@@ -11,6 +11,7 @@ services:
user: node
command: npm run test:unit:_run
environment:
CI:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
@@ -20,6 +21,7 @@ services:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
@@ -45,6 +47,7 @@ services:
GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
volumes:
- ./reports:/overleaf/services/filestore/reports
- minio-certs:/certs
depends_on:
certs:
services/filestore/package.json
+6 -4
View File
@@ -4,17 +4,17 @@
"private": true,
"main": "app.js",
"scripts": {
"test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000 $@ test/acceptance/js",
"test:acceptance:run": "mocha --recursive --timeout 15000 $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:run": "mocha --recursive --reporter spec $@ test/unit/js",
"test:unit:run": "mocha --recursive $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"start": "node app.js",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
"format": "prettier --list-different $PWD/'**/{*.*js,*.ts}'",
"format:fix": "prettier --write $PWD/'**/{*.*js,*.ts}'",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
@@ -40,6 +40,8 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"mongodb": "6.12.0",
"sandboxed-module": "2.0.4",
"sinon": "9.0.2",
services/filestore/test/mocha-multi-reporters.cjs
+8
View File
@@ -0,0 +1,8 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
},
}
services/history-v1/.mocharc.cjs
+13
View File
@@ -0,0 +1,13 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: '/overleaf/node_modules/mocha-multi-reporters',
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
services/history-v1/.mocharc.json
-3
View File
@@ -1,3 +0,0 @@
{
"require": "test/setup.js"
}
services/history-v1/Dockerfile
+23 -3
View File
@@ -18,12 +18,32 @@ RUN mkdir /buckets && chown node:node /buckets
FROM base AS app
COPY package.json package-lock.json /overleaf/
COPY services/history-v1/package.json /overleaf/services/history-v1/
COPY libraries/ /overleaf/libraries/
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/object-persistor/package.json /overleaf/libraries/object-persistor/package.json
COPY libraries/overleaf-editor-core/package.json /overleaf/libraries/overleaf-editor-core/package.json
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
COPY services/history-v1/package.json /overleaf/services/history-v1/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/object-persistor/ /overleaf/libraries/object-persistor/
COPY libraries/overleaf-editor-core/ /overleaf/libraries/overleaf-editor-core/
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY services/history-v1/ /overleaf/services/history-v1/
FROM app
services/history-v1/Jenkinsfile
Vendored
+125
View File
@@ -0,0 +1,125 @@
// Autogenerated by build scripts. Do not edit.
pipeline {
agent {
node {
label 'jenkins-agent-web'
customWorkspace '/workspace'
}
}
options {
timestamps()
timeout(time: 15, unit: 'MINUTES')
}
environment {
BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}"
JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}"
BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}"
DOCKER_COMPOSE_FLAGS = '-f docker-compose.ci.yml'
COMMIT_SHA = "${GIT_COMMIT}"
SHORT_SHA = "${GIT_COMMIT.take(7)}"
}
stages {
stage('Stage 1') {
parallel {
stage('Build') {
steps {
dir('services/history-v1') {
retry(count: 3) {
sh 'make build'
}
}
}
}
stage('Create reports folder') {
steps {
sh 'mkdir services/history-v1/reports'
}
}
}
}
stage('Stage 2') {
parallel {
stage('Push Branch Image') {
steps {
dir('services/history-v1') {
sh 'make push_branch'
}
}
}
stage('Shellcheck') {
steps {
dir('services/history-v1') {
sh 'make shellcheck'
}
}
}
stage('Lint') {
steps {
dir('services/history-v1') {
sh 'make lint_ci'
}
}
post {
always {
recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'history-v1-eslint', name: 'history-v1 eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'services/history-v1/reports/eslint.json')]
}
}
}
stage('Format') {
steps {
dir('services/history-v1') {
sh 'make format_ci'
}
}
}
stage('Typecheck') {
steps {
dir('services/history-v1') {
sh 'make typecheck_ci'
}
}
}
stage('Test Unit') {
steps {
dir('services/history-v1') {
retry(count: 3) {
sh 'make test_unit'
}
}
}
}
stage('Test Acceptance') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance"
}
steps {
dir('services/history-v1') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
}
}
stage('Push Production') {
steps {
dir('services/history-v1') {
sh 'make push'
}
}
}
}
post {
always {
junit checksName: 'history-v1 test results', testResults: 'services/history-v1/reports/junit-*.xml'
}
cleanup {
dir('services/history-v1') {
sh 'make clean'
}
sh 'make clean_jenkins -j10'
}
}
}
services/history-v1/Makefile
+9 -6
View File
@@ -27,14 +27,15 @@ clean:
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
-rm -rf reports/
HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent
@@ -70,7 +71,8 @@ lint:
$(RUN_LINTING) lint
lint_ci:
$(RUN_LINTING_CI) lint
-$(RUN_LINTING_CI) lint -- --format json --output-file reports/eslint.json
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
lint_fix:
$(RUN_LINTING) lint:fix
@@ -137,10 +139,11 @@ build:
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push_branch:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)
.PHONY: clean \
format format_fix \
services/history-v1/docker-compose.ci.yml
+4
View File
@@ -10,10 +10,12 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
volumes:
- ./reports:/overleaf/services/history-v1/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 --
command: npm run test:unit:_run
environment:
CI:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
@@ -32,6 +34,7 @@ services:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
@@ -53,6 +56,7 @@ services:
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
volumes:
- ./reports:/overleaf/services/history-v1/reports
- minio-certs:/certs
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
services/history-v1/package.json
+4 -2
View File
@@ -52,6 +52,8 @@
"chai-as-promised": "^7.1.1",
"chai-exclude": "^2.1.1",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"node-fetch": "^2.7.0",
"sinon": "^9.0.2",
"swagger-client": "^3.10.0",
@@ -66,8 +68,8 @@
"format:fix": "prettier --write $PWD/'**/{*.*js,*.ts}'",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"nodemon": "node --watch app.js",
"migrate": "knex migrate:latest",
"delete_old_chunks": "node storage/tasks/delete_old_chunks.js",
services/history-v1/test/mocha-multi-reporters.cjs
+8
View File
@@ -0,0 +1,8 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
},
}
services/notifications/.mocharc.json
-3
View File
@@ -1,3 +0,0 @@
{
"require": "test/setup.js"
}
services/notifications/Dockerfile
+17 -3
View File
@@ -13,12 +13,26 @@ RUN mkdir /home/node/.config && chown node:node /home/node/.config
FROM base AS app
COPY package.json package-lock.json /overleaf/
COPY services/notifications/package.json /overleaf/services/notifications/
COPY libraries/ /overleaf/libraries/
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
COPY services/notifications/package.json /overleaf/services/notifications/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
COPY services/notifications/ /overleaf/services/notifications/
FROM app
services/notifications/Jenkinsfile
Vendored
+125
View File
@@ -0,0 +1,125 @@
// Autogenerated by build scripts. Do not edit.
pipeline {
agent {
node {
label 'jenkins-agent-web'
customWorkspace '/workspace'
}
}
options {
timestamps()
timeout(time: 15, unit: 'MINUTES')
}
environment {
BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}"
JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}"
BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}"
DOCKER_COMPOSE_FLAGS = '-f docker-compose.ci.yml'
COMMIT_SHA = "${GIT_COMMIT}"
SHORT_SHA = "${GIT_COMMIT.take(7)}"
}
stages {
stage('Stage 1') {
parallel {
stage('Build') {
steps {
dir('services/notifications') {
retry(count: 3) {
sh 'make build'
}
}
}
}
stage('Create reports folder') {
steps {
sh 'mkdir services/notifications/reports'
}
}
}
}
stage('Stage 2') {
parallel {
stage('Push Branch Image') {
steps {
dir('services/notifications') {
sh 'make push_branch'
}
}
}
stage('Shellcheck') {
steps {
dir('services/notifications') {
sh 'make shellcheck'
}
}
}
stage('Lint') {
steps {
dir('services/notifications') {
sh 'make lint_ci'
}
}
post {
always {
recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'notifications-eslint', name: 'notifications eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'services/notifications/reports/eslint.json')]
}
}
}
stage('Format') {
steps {
dir('services/notifications') {
sh 'make format_ci'
}
}
}
stage('Typecheck') {
steps {
dir('services/notifications') {
sh 'make typecheck_ci'
}
}
}
stage('Test Unit') {
steps {
dir('services/notifications') {
retry(count: 3) {
sh 'make test_unit'
}
}
}
}
stage('Test Acceptance') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance"
}
steps {
dir('services/notifications') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
}
}
stage('Push Production') {
steps {
dir('services/notifications') {
sh 'make push'
}
}
}
}
post {
always {
junit checksName: 'notifications test results', testResults: 'services/notifications/reports/junit-*.xml'
}
cleanup {
dir('services/notifications') {
sh 'make clean'
}
sh 'make clean_jenkins -j10'
}
}
}
services/notifications/Makefile
+9 -6
View File
@@ -27,14 +27,15 @@ clean:
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
-rm -rf reports/
HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent
@@ -70,7 +71,8 @@ lint:
$(RUN_LINTING) lint
lint_ci:
$(RUN_LINTING_CI) lint
-$(RUN_LINTING_CI) lint -- --format json --output-file reports/eslint.json
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
lint_fix:
$(RUN_LINTING) lint:fix
@@ -137,10 +139,11 @@ build:
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push_branch:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)
.PHONY: clean \
format format_fix \
services/notifications/docker-compose.ci.yml
+4
View File
@@ -7,11 +7,13 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
volumes:
- ./reports:/overleaf/services/notifications/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 --
command: npm run test:unit:_run
environment:
CI:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
@@ -25,6 +27,7 @@ services:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
@@ -32,6 +35,7 @@ services:
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
volumes:
- ./reports:/overleaf/services/notifications/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
depends_on:
services/notifications/vitest.config.acceptance.cjs
+16
View File
@@ -1,8 +1,24 @@
const { defineConfig } = require('vitest/config')
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporters: [
'default',
[
'junit',
{
classnameTemplate: `Acceptance tests.{filename}`,
},
],
],
outputFile: 'reports/junit-vitest-acceptance.xml',
}
}
module.exports = defineConfig({
test: {
include: ['test/acceptance/js/**/*.test.{js,ts}'],
isolate: false,
...reporterOptions,
},
})
services/notifications/vitest.config.unit.cjs
+16
View File
@@ -1,9 +1,25 @@
const { defineConfig } = require('vitest/config')
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporters: [
'default',
[
'junit',
{
classnameTemplate: `Unit tests.{filename}`,
},
],
],
outputFile: 'reports/junit-vitest-unit.xml',
}
}
module.exports = defineConfig({
test: {
include: ['test/unit/js/**/*.test.{js,ts}'],
setupFiles: ['./test/setup.js'],
isolate: false,
...reporterOptions,
},
})
services/project-history/.mocharc.cjs
+13
View File
@@ -0,0 +1,13 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: '/overleaf/node_modules/mocha-multi-reporters',
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
services/project-history/.mocharc.json
-3
View File
@@ -1,3 +0,0 @@
{
"require": "test/setup.js"
}
services/project-history/Dockerfile
+21 -3
View File
@@ -13,12 +13,30 @@ RUN mkdir /home/node/.config && chown node:node /home/node/.config
FROM base AS app
COPY package.json package-lock.json /overleaf/
COPY services/project-history/package.json /overleaf/services/project-history/
COPY libraries/ /overleaf/libraries/
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/overleaf-editor-core/package.json /overleaf/libraries/overleaf-editor-core/package.json
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
COPY services/project-history/package.json /overleaf/services/project-history/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/overleaf-editor-core/ /overleaf/libraries/overleaf-editor-core/
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY services/project-history/ /overleaf/services/project-history/
FROM app
services/project-history/Jenkinsfile
Vendored
+125
View File
@@ -0,0 +1,125 @@
// Autogenerated by build scripts. Do not edit.
pipeline {
agent {
node {
label 'jenkins-agent-web'
customWorkspace '/workspace'
}
}
options {
timestamps()
timeout(time: 15, unit: 'MINUTES')
}
environment {
BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}"
JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}"
BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}"
DOCKER_COMPOSE_FLAGS = '-f docker-compose.ci.yml'
COMMIT_SHA = "${GIT_COMMIT}"
SHORT_SHA = "${GIT_COMMIT.take(7)}"
}
stages {
stage('Stage 1') {
parallel {
stage('Build') {
steps {
dir('services/project-history') {
retry(count: 3) {
sh 'make build'
}
}
}
}
stage('Create reports folder') {
steps {
sh 'mkdir services/project-history/reports'
}
}
}
}
stage('Stage 2') {
parallel {
stage('Push Branch Image') {
steps {
dir('services/project-history') {
sh 'make push_branch'
}
}
}
stage('Shellcheck') {
steps {
dir('services/project-history') {
sh 'make shellcheck'
}
}
}
stage('Lint') {
steps {
dir('services/project-history') {
sh 'make lint_ci'
}
}
post {
always {
recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'project-history-eslint', name: 'project-history eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'services/project-history/reports/eslint.json')]
}
}
}
stage('Format') {
steps {
dir('services/project-history') {
sh 'make format_ci'
}
}
}
stage('Typecheck') {
steps {
dir('services/project-history') {
sh 'make typecheck_ci'
}
}
}
stage('Test Unit') {
steps {
dir('services/project-history') {
retry(count: 3) {
sh 'make test_unit'
}
}
}
}
stage('Test Acceptance') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance"
}
steps {
dir('services/project-history') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
}
}
stage('Push Production') {
steps {
dir('services/project-history') {
sh 'make push'
}
}
}
}
post {
always {
junit checksName: 'project-history test results', testResults: 'services/project-history/reports/junit-*.xml'
}
cleanup {
dir('services/project-history') {
sh 'make clean'
}
sh 'make clean_jenkins -j10'
}
}
}
services/project-history/Makefile
+9 -6
View File
@@ -27,14 +27,15 @@ clean:
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
-rm -rf reports/
HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent
@@ -70,7 +71,8 @@ lint:
$(RUN_LINTING) lint
lint_ci:
$(RUN_LINTING_CI) lint
-$(RUN_LINTING_CI) lint -- --format json --output-file reports/eslint.json
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
lint_fix:
$(RUN_LINTING) lint:fix
@@ -137,10 +139,11 @@ build:
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push_branch:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)
.PHONY: clean \
format format_fix \
services/project-history/docker-compose.ci.yml
+4
View File
@@ -7,10 +7,12 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
volumes:
- ./reports:/overleaf/services/project-history/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 --
command: npm run test:unit:_run
environment:
CI:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
@@ -29,6 +31,7 @@ services:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
@@ -40,6 +43,7 @@ services:
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
volumes:
- ./reports:/overleaf/services/project-history/reports
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
services/project-history/package.json
+4 -2
View File
@@ -9,8 +9,8 @@
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"start": "node app.js",
"nodemon": "node --watch app.js",
"test:acceptance:_run": "mocha --loader=esmock --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --loader=esmock --recursive --reporter spec --exit $@ test/unit/js",
"test:acceptance:_run": "mocha --loader=esmock --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --loader=esmock --recursive --exit $@ test/unit/js",
"lint": "eslint --max-warnings 0 --format unix .",
"format": "prettier --list-different $PWD/'**/{*.*js,*.ts}'",
"format:fix": "prettier --write $PWD/'**/{*.*js,*.ts}'",
@@ -45,6 +45,8 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"nock": "^13.5.3",
"sinon": "~9.0.1",
"sinon-chai": "^3.7.0",
services/project-history/test/mocha-multi-reporters.cjs
+8
View File
@@ -0,0 +1,8 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
},
}
services/real-time/.mocharc.cjs
+13
View File
@@ -0,0 +1,13 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: '/overleaf/node_modules/mocha-multi-reporters',
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
services/real-time/.mocharc.json
-3
View File
@@ -1,3 +0,0 @@
{
"require": "test/setup.js"
}
services/real-time/Dockerfile
+13 -3
View File
@@ -13,12 +13,22 @@ RUN mkdir /home/node/.config && chown node:node /home/node/.config
FROM base AS app
COPY package.json package-lock.json /overleaf/
COPY services/real-time/package.json /overleaf/services/real-time/
COPY libraries/ /overleaf/libraries/
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY services/real-time/package.json /overleaf/services/real-time/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY services/real-time/ /overleaf/services/real-time/
FROM app
services/real-time/Jenkinsfile
Vendored
+125
View File
@@ -0,0 +1,125 @@
// Autogenerated by build scripts. Do not edit.
pipeline {
agent {
node {
label 'jenkins-agent-web'
customWorkspace '/workspace'
}
}
options {
timestamps()
timeout(time: 15, unit: 'MINUTES')
}
environment {
BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}"
JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}"
BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}"
DOCKER_COMPOSE_FLAGS = '-f docker-compose.ci.yml'
COMMIT_SHA = "${GIT_COMMIT}"
SHORT_SHA = "${GIT_COMMIT.take(7)}"
}
stages {
stage('Stage 1') {
parallel {
stage('Build') {
steps {
dir('services/real-time') {
retry(count: 3) {
sh 'make build'
}
}
}
}
stage('Create reports folder') {
steps {
sh 'mkdir services/real-time/reports'
}
}
}
}
stage('Stage 2') {
parallel {
stage('Push Branch Image') {
steps {
dir('services/real-time') {
sh 'make push_branch'
}
}
}
stage('Shellcheck') {
steps {
dir('services/real-time') {
sh 'make shellcheck'
}
}
}
stage('Lint') {
steps {
dir('services/real-time') {
sh 'make lint_ci'
}
}
post {
always {
recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'real-time-eslint', name: 'real-time eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'services/real-time/reports/eslint.json')]
}
}
}
stage('Format') {
steps {
dir('services/real-time') {
sh 'make format_ci'
}
}
}
stage('Typecheck') {
steps {
dir('services/real-time') {
sh 'make typecheck_ci'
}
}
}
stage('Test Unit') {
steps {
dir('services/real-time') {
retry(count: 3) {
sh 'make test_unit'
}
}
}
}
stage('Test Acceptance') {
environment {
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE = "test_acceptance"
}
steps {
dir('services/real-time') {
retry(count: 3) {
sh 'make test_acceptance'
}
}
}
}
}
}
stage('Push Production') {
steps {
dir('services/real-time') {
sh 'make push'
}
}
}
}
post {
always {
junit checksName: 'real-time test results', testResults: 'services/real-time/reports/junit-*.xml'
}
cleanup {
dir('services/real-time') {
sh 'make clean'
}
sh 'make clean_jenkins -j10'
}
}
}
services/real-time/Makefile
+9 -6
View File
@@ -27,14 +27,15 @@ clean:
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
-rm -rf reports/
HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
export MONOREPO ?= $(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent
@@ -70,7 +71,8 @@ lint:
$(RUN_LINTING) lint
lint_ci:
$(RUN_LINTING_CI) lint
-$(RUN_LINTING_CI) lint -- --format json --output-file reports/eslint.json
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
lint_fix:
$(RUN_LINTING) lint:fix
@@ -137,10 +139,11 @@ build:
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
push_branch:
docker push us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)
.PHONY: clean \
format format_fix \
services/real-time/docker-compose.ci.yml
+4
View File
@@ -8,6 +8,7 @@ services:
user: node
command: npm run test:unit:_run
environment:
CI:
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
@@ -24,6 +25,7 @@ services:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
CI:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
@@ -34,6 +36,8 @@ services:
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
volumes:
- ./reports:/overleaf/services/real-time/reports
depends_on:
redis_test:
condition: service_healthy
services/real-time/package.json
+4 -2
View File
@@ -5,9 +5,9 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
@@ -42,6 +42,8 @@
"chai-as-promised": "^7.1.1",
"cookie-signature": "^1.1.0",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"sandboxed-module": "~0.3.0",
"sinon": "^9.2.4",
"sinon-chai": "^3.7.0",
services/real-time/test/mocha-multi-reporters.cjs
+8
View File
@@ -0,0 +1,8 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
},
}
services/web/Dockerfile
+52 -6
View File
@@ -1,5 +1,6 @@
# the base image is suitable for running web with /overleaf/services/web bind
# mounted
# This file was auto-generated, do not edit it directly.
# Instead run internal$ services/web/build_scripts
FROM node:22.18.0 AS base
WORKDIR /overleaf/services/web
@@ -22,8 +23,22 @@ RUN mkdir -p /overleaf/services/web/data/dumpFolder \
FROM base AS deps-prod
COPY package.json package-lock.json /overleaf/
COPY services/web/package.json /overleaf/services/web/
COPY libraries/ /overleaf/libraries/
COPY libraries/access-token-encryptor/package.json /overleaf/libraries/access-token-encryptor/package.json
COPY libraries/eslint-plugin/package.json /overleaf/libraries/eslint-plugin/package.json
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/object-persistor/package.json /overleaf/libraries/object-persistor/package.json
COPY libraries/overleaf-editor-core/package.json /overleaf/libraries/overleaf-editor-core/package.json
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
COPY libraries/ranges-tracker/package.json /overleaf/libraries/ranges-tracker/package.json
COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
COPY services/web/package.json /overleaf/services/web/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && NODE_ENV=production npm ci --quiet
@@ -41,7 +56,22 @@ FROM deps AS dev
ARG SENTRY_RELEASE
ENV SENTRY_RELEASE=$SENTRY_RELEASE
COPY services/web /overleaf/services/web
COPY libraries/access-token-encryptor/ /overleaf/libraries/access-token-encryptor/
COPY libraries/eslint-plugin/ /overleaf/libraries/eslint-plugin/
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/object-persistor/ /overleaf/libraries/object-persistor/
COPY libraries/overleaf-editor-core/ /overleaf/libraries/overleaf-editor-core/
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/ranges-tracker/ /overleaf/libraries/ranges-tracker/
COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
COPY services/web/ /overleaf/services/web/
# Build the latex parser
RUN cd /overleaf/services/web && npm run 'lezer-latex:generate'
@@ -62,7 +92,23 @@ RUN nice find /overleaf/services/web/public -name '*.js.map' -delete
# copy source code and precompile pug images
FROM deps-prod AS pug
COPY services/web /overleaf/services/web
COPY libraries/access-token-encryptor/ /overleaf/libraries/access-token-encryptor/
COPY libraries/eslint-plugin/ /overleaf/libraries/eslint-plugin/
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/object-persistor/ /overleaf/libraries/object-persistor/
COPY libraries/overleaf-editor-core/ /overleaf/libraries/overleaf-editor-core/
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/ranges-tracker/ /overleaf/libraries/ranges-tracker/
COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
COPY services/web/ /overleaf/services/web/
# Omit Server Pro/CE specific scripts from SaaS image
RUN rm /overleaf/services/web/modules/server-ce-scripts -rf
RUN OVERLEAF_CONFIG=/overleaf/services/web/config/settings.overrides.saas.js nice npm run precompile-pug
services/web/buildscript.txt
-1
View File
@@ -1,5 +1,4 @@
web
--only-jenkins-include-file=True
--dependencies=
--docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
--env-add=