diff --git a/server-ce/Makefile b/server-ce/Makefile index 853a99d05e..a095530f88 100644 --- a/server-ce/Makefile +++ b/server-ce/Makefile @@ -41,6 +41,12 @@ build-community: --tag $(OVERLEAF_BRANCH) \ $(MONOREPO_ROOT) +push: + docker push $(OVERLEAF_BASE_TAG) + docker push $(OVERLEAF_BASE_BRANCH) + docker push $(OVERLEAF_TAG) + docker push $(OVERLEAF_BRANCH) + SHELLCHECK_OPTS = \ --shell=bash \ --external-sources \ diff --git a/server-ce/test/.jenkinsIncludeFile b/server-ce/test/.jenkinsIncludeFile new file mode 100644 index 0000000000..6e634e756c --- /dev/null +++ b/server-ce/test/.jenkinsIncludeFile @@ -0,0 +1,154 @@ +copybara/** + +libraries/** + +patches/** + +server-ce/** +server-pro/** + +services/clsi/seccomp/** +services/history-v1/api/** +services/history-v1/storage/** + +# echo chat clsi contacts docstore document-updater filestore history-v1 notifications project-history real-time references templates | xargs -n1 echo | xargs -I% echo 'services/%/*' 'services/%/app/**' 'services/%/config/*' 'services/%/scripts/**' | xargs -n1 echo | sort +services/chat/* +services/chat/app/** +services/chat/config/* +services/chat/scripts/** +services/clsi/* +services/clsi/app/** +services/clsi/config/* +services/clsi/scripts/** +services/contacts/* +services/contacts/app/** +services/contacts/config/* +services/contacts/scripts/** +services/docstore/* +services/docstore/app/** +services/docstore/config/* +services/docstore/scripts/** +services/document-updater/* +services/document-updater/app/** +services/document-updater/config/* +services/document-updater/scripts/** +services/filestore/* +services/filestore/app/** +services/filestore/config/* +services/filestore/scripts/** +services/history-v1/* +services/history-v1/app/** +services/history-v1/config/* +services/history-v1/scripts/** +services/notifications/* +services/notifications/app/** +services/notifications/config/* +services/notifications/scripts/** +services/project-history/* +services/project-history/app/** +services/project-history/config/* +services/project-history/scripts/** +services/real-time/* +services/real-time/app/** +services/real-time/config/* +services/real-time/scripts/** +services/references/* +services/references/app/** +services/references/config/* +services/references/scripts/** +services/templates/* +services/templates/app/** +services/templates/config/* +services/templates/scripts/** + + +services/web/* +services/web/app/** +services/web/config/settings.defaults.js +services/web/config/settings.overrides.server-pro.js +services/web/frontend/** +services/web/locales/** +services/web/migrations/** + +# echo admin-panel admin-roles full-project-search git-bridge lauchpad ldap learn oauth2-server server-ce-scripts server-pro-saml symbol-palette templates track-changes user-activate | xargs -n1 echo | xargs -I% echo 'services/web/modules/%/*' 'services/web/modules/%/app/**' 'services/web/modules/%/frontend/**' 'services/web/modules/%/scripts/**' 'services/web/modules/%/types/**' | xargs -n1 echo | sort +services/web/modules/admin-panel/* +services/web/modules/admin-panel/app/** +services/web/modules/admin-panel/frontend/** +services/web/modules/admin-panel/scripts/** +services/web/modules/admin-panel/types/** +services/web/modules/admin-roles/* +services/web/modules/admin-roles/app/** +services/web/modules/admin-roles/frontend/** +services/web/modules/admin-roles/scripts/** +services/web/modules/admin-roles/types/** +services/web/modules/full-project-search/* +services/web/modules/full-project-search/app/** +services/web/modules/full-project-search/frontend/** +services/web/modules/full-project-search/scripts/** +services/web/modules/full-project-search/types/** +services/web/modules/git-bridge/* +services/web/modules/git-bridge/app/** +services/web/modules/git-bridge/frontend/** +services/web/modules/git-bridge/scripts/** +services/web/modules/git-bridge/types/** +services/web/modules/lauchpad/* +services/web/modules/lauchpad/app/** +services/web/modules/lauchpad/frontend/** +services/web/modules/lauchpad/scripts/** +services/web/modules/lauchpad/types/** +services/web/modules/ldap/* +services/web/modules/ldap/app/** +services/web/modules/ldap/frontend/** +services/web/modules/ldap/scripts/** +services/web/modules/ldap/types/** +services/web/modules/learn/* +services/web/modules/learn/app/** +services/web/modules/learn/frontend/** +services/web/modules/learn/scripts/** +services/web/modules/learn/types/** +services/web/modules/oauth2-server/* +services/web/modules/oauth2-server/app/** +services/web/modules/oauth2-server/frontend/** +services/web/modules/oauth2-server/scripts/** +services/web/modules/oauth2-server/types/** +services/web/modules/server-ce-scripts/* +services/web/modules/server-ce-scripts/app/** +services/web/modules/server-ce-scripts/frontend/** +services/web/modules/server-ce-scripts/scripts/** +services/web/modules/server-ce-scripts/types/** +services/web/modules/server-pro-saml/* +services/web/modules/server-pro-saml/app/** +services/web/modules/server-pro-saml/frontend/** +services/web/modules/server-pro-saml/scripts/** +services/web/modules/server-pro-saml/types/** +services/web/modules/symbol-palette/* +services/web/modules/symbol-palette/app/** +services/web/modules/symbol-palette/frontend/** +services/web/modules/symbol-palette/scripts/** +services/web/modules/symbol-palette/types/** +services/web/modules/templates/* +services/web/modules/templates/app/** +services/web/modules/templates/frontend/** +services/web/modules/templates/scripts/** +services/web/modules/templates/types/** +services/web/modules/track-changes/* +services/web/modules/track-changes/app/** +services/web/modules/track-changes/frontend/** +services/web/modules/track-changes/scripts/** +services/web/modules/track-changes/types/** +services/web/modules/user-activate/* +services/web/modules/user-activate/app/** +services/web/modules/user-activate/frontend/** +services/web/modules/user-activate/scripts/** +services/web/modules/user-activate/types/** + +services/web/public/** +services/web/types/** +services/web/webpack-plugins/** + +.dockerignore +.eslint* +.pretter* +package.json +package-lock.json +tsconfig.backend.json diff --git a/server-ce/test/Jenkinsfile b/server-ce/test/Jenkinsfile new file mode 100644 index 0000000000..e785970207 --- /dev/null +++ b/server-ce/test/Jenkinsfile @@ -0,0 +1,222 @@ +def job_copybara_done = false +def job_npm_install_done = false +def job_prefetch_custom_done = false +def job_prefetch_default_done = false +def job_server_ce_build_done = false +def job_server_pro_build_done = false + +pipeline { + agent { + node { + label 'jenkins-agent-web' + customWorkspace '/workspace' + } + } + + options { + timestamps() + } + + environment { + BRANCH_NAME = "${GIT_BRANCH.replace('origin/', '')}" + COMMIT_SHA = "${GIT_COMMIT}" + SHORT_SHA = "${GIT_COMMIT.take(7)}" + + OVERLEAF_BASE_BRANCH = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base-internal:${BRANCH_NAME}" + OVERLEAF_BASE_LATEST = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base-internal:main" + OVERLEAF_BASE_TAG = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base-internal:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}" + OVERLEAF_BRANCH = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-internal:${BRANCH_NAME}" + OVERLEAF_LATEST = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-internal:main" + OVERLEAF_TAG = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-internal:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}" + IMAGE_TAG_CE = "${OVERLEAF_TAG}" + IMAGE_TAG_PRO = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pro:main" + } + + stages { + stage('Parallel') { + parallel { + stage('Install deps') { + steps { + sh 'make install' + sh 'make -C server-ce/test npm_install_in_docker' + script { job_npm_install_done = true } + } + } + + stage('shellcheck') { + steps { + dir('server-ce') { + sh 'make shellcheck' + } + } + } + + stage('Format') { + steps { + script { waitUntil { return job_npm_install_done } } + dir('server-ce/test') { + sh 'make format_in_docker' + } + } + } + + stage('Copybara') { + steps { + sh 'copybara/bin/sync' + script { job_copybara_done = true } + } + } + + stage('Build CE image') { + steps { + script { waitUntil { return job_copybara_done } } + dir('copybara/public/repo/server-ce') { + sh 'make build-base' + sh 'make build-community' + } + script { job_server_ce_build_done = true } + } + } + stage('Push CE to internal') { + steps { + script { waitUntil { return job_server_ce_build_done } } + dir('copybara/public/repo/server-ce') { + sh 'make push' + } + } + } + + stage('Build Pro image') { + environment { + OVERLEAF_CE_TAG = "${OVERLEAF_TAG}" + OVERLEAF_PRO_TAG= "${IMAGE_TAG_PRO}" + } + steps { + script { waitUntil { return job_server_ce_build_done } } + dir('server-pro') { + sh 'make build-ci' + } + script { job_server_pro_build_done = true } + } + } + + stage('Prefetch default') { + steps { + dir('server-ce/test') { + sh 'make prefetch_default -j4' + } + script { job_prefetch_default_done = true } + } + } + stage('Prefetch custom') { + steps { + dir('server-ce/test') { + sh 'make prefetch_custom -j4' + } + script { job_prefetch_custom_done = true } + } + } + + stage('CE default') { + environment { + CYPRESS_SHARD = "CE_DEFAULT" + COMPOSE_PROJECT_NAME = "test-ce-default" + } + steps { + script { waitUntil { return job_npm_install_done && job_server_ce_build_done && job_prefetch_default_done } } + dir('server-ce/test') { + sh 'make test-e2e' + } + } + } + stage('CE custom 1') { + environment { + CYPRESS_SHARD = "CE_CUSTOM_1" + COMPOSE_PROJECT_NAME = "test-ce-custom-1" + } + steps { + script { waitUntil { return job_npm_install_done && job_server_ce_build_done && job_prefetch_default_done && job_prefetch_custom_done } } + dir('server-ce/test') { + sh 'make test-e2e' + } + } + } + stage('CE custom 2') { + environment { + CYPRESS_SHARD = "CE_CUSTOM_2" + COMPOSE_PROJECT_NAME = "test-ce-custom-2" + } + steps { + script { waitUntil { return job_npm_install_done && job_server_ce_build_done && job_prefetch_default_done && job_prefetch_custom_done } } + dir('server-ce/test') { + sh 'make test-e2e' + } + } + } + + stage('PRO default 1') { + environment { + CYPRESS_SHARD = "PRO_DEFAULT_1" + COMPOSE_PROJECT_NAME = "test-pro-default-1" + } + steps { + script { waitUntil { return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done } } + dir('server-ce/test') { + sh 'make test-e2e' + } + } + } + stage('PRO default 2') { + environment { + CYPRESS_SHARD = "PRO_DEFAULT_2" + COMPOSE_PROJECT_NAME = "test-pro-default-2" + } + steps { + script { waitUntil { return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done } } + dir('server-ce/test') { + sh 'make test-e2e' + } + } + } + stage('PRO custom 1') { + environment { + CYPRESS_SHARD = "PRO_CUSTOM_1" + COMPOSE_PROJECT_NAME = "test-pro-custom-1" + } + steps { + script { waitUntil { return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done && job_prefetch_custom_done } } + dir('server-ce/test') { + sh 'make test-e2e' + } + } + } + stage('PRO custom 2') { + environment { + CYPRESS_SHARD = "PRO_CUSTOM_2" + COMPOSE_PROJECT_NAME = "test-pro-custom-2" + } + steps { + script { waitUntil { return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done && job_prefetch_custom_done } } + dir('server-ce/test') { + sh 'make test-e2e' + } + } + } + stage('PRO custom 3') { + environment { + CYPRESS_SHARD = "PRO_CUSTOM_3" + COMPOSE_PROJECT_NAME = "test-pro-custom-3" + } + steps { + script { waitUntil { return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done && job_prefetch_custom_done } } + dir('server-ce/test') { + sh 'make test-e2e' + } + } + } + } + } + } +} + +// vim: set ft=groovy : diff --git a/server-ce/test/Makefile b/server-ce/test/Makefile index 51e953ca89..10c7bb49e5 100644 --- a/server-ce/test/Makefile +++ b/server-ce/test/Makefile @@ -22,11 +22,13 @@ test-e2e-native: test-e2e: docker compose build host-admin - docker compose up -d host-admin - docker compose up --no-log-prefix --exit-code-from=e2e e2e + docker compose up --detach --wait host-admin + docker compose up --detach --wait mongo + docker compose up --no-log-prefix --exit-code-from=e2e e2e host-admin test-e2e-open: - docker compose up -d host-admin + docker compose up --detach --wait host-admin + docker compose up --detach --wait mongo docker compose up --no-log-prefix --exit-code-from=e2e-open e2e-open host-admin clean: @@ -58,12 +60,12 @@ prefetch_custom_texlive_2022: prefetch_custom: prefetch_old_4_2 prefetch_old_4_2: - docker pull $(IMAGE_TAG_PRO:latest=4.2) + docker pull $(IMAGE_TAG_PRO:main=4.2) prefetch_custom: prefetch_old_5_0 prefetch_old_5_0: - docker pull $(IMAGE_TAG_PRO:latest=5.0.1-RC1) - docker pull $(IMAGE_TAG_PRO:latest=5.0) + docker pull $(IMAGE_TAG_PRO:main=5.0.1-RC1) + docker pull $(IMAGE_TAG_PRO:main=5.0) # Google Cloud Build runs on a very ancient Docker version that does not support the subdir flag. # Use services -> mailtrap -> build -> context = https://github.com/dbck/docker-mailtrap.git#v1.5.0:build in docker-compose.yml eventually. diff --git a/server-ce/test/docker-compose.yml b/server-ce/test/docker-compose.yml index 145fb7d7dd..6d3728af6a 100644 --- a/server-ce/test/docker-compose.yml +++ b/server-ce/test/docker-compose.yml @@ -103,6 +103,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock stop_grace_period: 0s environment: + CI: PWD: CYPRESS_SHARD: COMPOSE_PROJECT_NAME: @@ -110,9 +111,6 @@ services: ALL_TEX_LIVE_DOCKER_IMAGES: IMAGE_TAG_CE: ${IMAGE_TAG_CE:-sharelatex/sharelatex:latest} IMAGE_TAG_PRO: ${IMAGE_TAG_PRO:-quay.io/sharelatex/sharelatex-pro:latest} - depends_on: - mongo: - condition: service_healthy healthcheck: test: curl --fail http://localhost/status interval: 3s diff --git a/server-ce/test/helpers/config.ts b/server-ce/test/helpers/config.ts index f11a4491aa..9a2236d602 100644 --- a/server-ce/test/helpers/config.ts +++ b/server-ce/test/helpers/config.ts @@ -10,7 +10,6 @@ export function isExcludedBySharding( | 'CE_DEFAULT' | 'CE_CUSTOM_1' | 'CE_CUSTOM_2' - | 'CE_CUSTOM_3' | 'PRO_DEFAULT_1' | 'PRO_DEFAULT_2' | 'PRO_CUSTOM_1' diff --git a/server-ce/test/host-admin.js b/server-ce/test/host-admin.js index ee695058bf..4d55e68d7d 100644 --- a/server-ce/test/host-admin.js +++ b/server-ce/test/host-admin.js @@ -28,6 +28,11 @@ const IMAGES = { CE: process.env.IMAGE_TAG_CE.replace(/:.+/, ''), PRO: process.env.IMAGE_TAG_PRO.replace(/:.+/, ''), } +const LATEST = { + CE: process.env.IMAGE_TAG_CE.replace(/.+:/, '') || 'latest', + PRO: process.env.IMAGE_TAG_PRO.replace(/.+:/, '') || 'latest', + GIT_BRIDGE: 'latest', // TODO, build in CI? +} function defaultDockerComposeOverride() { return { @@ -80,10 +85,14 @@ app.get('/status', (req, res) => { app.use(bodyParser.json()) app.use((req, res, next) => { // Basic access logs - console.log(req.method, req.url, req.body) + if (process.env.CI !== 'true') { + console.log(req.method, req.url, req.body) + } const json = res.json res.json = body => { - console.log(req.method, req.url, req.body, '->', body) + if (process.env.CI !== 'true' || body.error) { + console.log(req.method, req.url, req.body, '->', body) + } json.call(res, body) } next() @@ -230,8 +239,9 @@ function setVarsDockerCompose({ }) { const cfg = readDockerComposeOverride() - cfg.services.sharelatex.image = `${pro ? IMAGES.PRO : IMAGES.CE}:${version}` - cfg.services['git-bridge'].image = `quay.io/sharelatex/git-bridge:${version}` + cfg.services.sharelatex.image = `${pro ? IMAGES.PRO : IMAGES.CE}:${version === 'latest' ? (pro ? LATEST.PRO : LATEST.CE) : version}` + cfg.services['git-bridge'].image = + `quay.io/sharelatex/git-bridge:${version === 'latest' ? LATEST.GIT_BRIDGE : version}` cfg.services.sharelatex.environment = vars