diff --git a/server-ce/hotfix/5.5.4/Dockerfile b/server-ce/hotfix/5.5.4/Dockerfile new file mode 100644 index 0000000000..0b0690c0cb --- /dev/null +++ b/server-ce/hotfix/5.5.4/Dockerfile @@ -0,0 +1,10 @@ +FROM sharelatex/sharelatex:5.5.3 + +# ../../bin/import_pr_patch.sh 27504 27505 +# Remove CE tests +# Remove web tests +# Fix cron paths +# Move init/shutdown/settings into .patch-abs file +COPY *.patch* . +RUN bash -ec 'for p in *.patch; do echo "=== Applying $p ==="; patch -p1 < "$p" && rm $p; done' \ +&& bash -ec 'cd / && for p in /overleaf/*.patch-abs; do echo "=== Applying $p ==="; patch -p1 < "$p" && rm $p; done' diff --git a/server-ce/hotfix/5.5.4/pr_27504.patch b/server-ce/hotfix/5.5.4/pr_27504.patch new file mode 100644 index 0000000000..76583666b8 --- /dev/null +++ b/server-ce/hotfix/5.5.4/pr_27504.patch @@ -0,0 +1,34 @@ + + +diff --git a/bin/flush-history-queues b/bin/flush-history-queues +index b54bc5558c6e..6c0cd896416e 100755 +--- a/bin/flush-history-queues ++++ b/bin/flush-history-queues +@@ -5,4 +5,4 @@ set -euo pipefail + source /etc/container_environment.sh + source /etc/overleaf/env.sh + cd /overleaf/services/project-history +-node scripts/flush_all.js 100000 ++exec /sbin/setuser www-data node scripts/flush_all.js 100000 +diff --git a/bin/force-history-resyncs b/bin/force-history-resyncs +index 389c98a4ad69..4f48890855b7 100755 +--- a/bin/force-history-resyncs ++++ b/bin/force-history-resyncs +@@ -5,4 +5,4 @@ set -euo pipefail + source /etc/container_environment.sh + source /etc/overleaf/env.sh + cd /overleaf/services/project-history +-node scripts/force_resync.js 1000 force ++exec /sbin/setuser www-data node scripts/force_resync.js 1000 force +diff --git a/cron/project-history-flush-all.sh b/cron/project-history-flush-all.sh +index 8fe9eea5fc55..689561737914 100755 +--- a/cron/project-history-flush-all.sh ++++ b/cron/project-history-flush-all.sh +@@ -9,6 +9,6 @@ date + + source /etc/container_environment.sh + source /etc/overleaf/env.sh +-cd /overleaf/services/project-history && node scripts/flush_all.js ++cd /overleaf/services/project-history && /sbin/setuser www-data node scripts/flush_all.js + + echo "Done flushing all project-history changes" diff --git a/server-ce/hotfix/5.5.4/pr_27504.patch-abs b/server-ce/hotfix/5.5.4/pr_27504.patch-abs new file mode 100644 index 0000000000..fb60c9f747 --- /dev/null +++ b/server-ce/hotfix/5.5.4/pr_27504.patch-abs @@ -0,0 +1,117 @@ +diff --git a/usr/local/bin/grunt b/usr/local/bin/grunt +index 462c68df4d52..8595d67109ae 100755 +--- a/usr/local/bin/grunt ++++ b/usr/local/bin/grunt +@@ -11,22 +11,22 @@ cd /overleaf/services/web + case "$TASK" in + user:create-admin) + echo "The grunt command is deprecated, run the create-user script using node instead" +- node modules/server-ce-scripts/scripts/create-user.mjs --admin "$@" ++ exec /sbin/setuser www-data node modules/server-ce-scripts/scripts/create-user.mjs --admin "$@" + ;; + + user:delete) + echo "The grunt command is deprecated, run the delete-user script using node instead" +- node modules/server-ce-scripts/scripts/delete-user.mjs "$@" ++ exec /sbin/setuser www-data node modules/server-ce-scripts/scripts/delete-user.mjs "$@" + ;; + + check:mongo) + echo "The grunt command is deprecated, run the check-mongodb script using node instead" +- node modules/server-ce-scripts/scripts/check-mongodb.mjs ++ exec /sbin/setuser www-data node modules/server-ce-scripts/scripts/check-mongodb.mjs + ;; + + check:redis) + echo "The grunt command is deprecated, run the check-redis script using node instead" +- node modules/server-ce-scripts/scripts/check-redis.mjs ++ exec /sbin/setuser www-data node modules/server-ce-scripts/scripts/check-redis.mjs + ;; + + *) +diff --git a/etc/my_init.pre_shutdown.d/00_close_site b/etc/my_init.pre_shutdown.d/00_close_site +index ed5404f8172f..ac579f4b10ff 100755 +--- a/etc/my_init.pre_shutdown.d/00_close_site ++++ b/etc/my_init.pre_shutdown.d/00_close_site +@@ -12,7 +12,7 @@ echo "closed" > "${SITE_MAINTENANCE_FILE}" + sleep 5 + + # giving a grace period of 5 seconds for users before disconnecting them and start shutting down +-cd /overleaf/services/web && node scripts/disconnect_all_users.mjs --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1 ++cd /overleaf/services/web && /sbin/setuser www-data node scripts/disconnect_all_users.mjs --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1 + + EXIT_CODE="$?" + if [ $EXIT_CODE -ne 0 ] +diff --git a/etc/my_init.pre_shutdown.d/01_flush_document_updater b/etc/my_init.pre_shutdown.d/01_flush_document_updater +index 0900fe5fac04..b4529f856b1e 100755 +--- a/etc/my_init.pre_shutdown.d/01_flush_document_updater ++++ b/etc/my_init.pre_shutdown.d/01_flush_document_updater +@@ -3,7 +3,7 @@ + . /etc/container_environment.sh + . /etc/overleaf/env.sh + +-cd /overleaf/services/document-updater && node scripts/flush_all.js >> /var/log/overleaf/document-updater.log 2>&1 ++cd /overleaf/services/document-updater && /sbin/setuser www-data node scripts/flush_all.js >> /var/log/overleaf/document-updater.log 2>&1 + + EXIT_CODE="$?" + if [ $EXIT_CODE -ne 0 ] +diff --git a/etc/my_init.pre_shutdown.d/02_flush_project_history b/etc/my_init.pre_shutdown.d/02_flush_project_history +index f8ac51600c98..2844379cebc1 100755 +--- a/etc/my_init.pre_shutdown.d/02_flush_project_history ++++ b/etc/my_init.pre_shutdown.d/02_flush_project_history +@@ -3,7 +3,7 @@ + . /etc/container_environment.sh + . /etc/overleaf/env.sh + +-cd /overleaf/services/project-history && node scripts/flush_all.js >> /var/log/overleaf/project-history.log 2>&1 ++cd /overleaf/services/project-history && /sbin/setuser www-data node scripts/flush_all.js >> /var/log/overleaf/project-history.log 2>&1 + + EXIT_CODE="$?" + if [ $EXIT_CODE -ne 0 ] +diff --git a/etc/my_init.d/500_check_db_access.sh b/etc/my_init.d/500_check_db_access.sh +index bbf2b9ec267d..f71acc8e011e 100755 +--- a/etc/my_init.d/500_check_db_access.sh ++++ b/etc/my_init.d/500_check_db_access.sh +@@ -3,6 +3,6 @@ set -e + + echo "Checking can connect to mongo and redis" + cd /overleaf/services/web +-node modules/server-ce-scripts/scripts/check-mongodb.mjs +-node modules/server-ce-scripts/scripts/check-redis.mjs ++/sbin/setuser www-data node modules/server-ce-scripts/scripts/check-mongodb.mjs ++/sbin/setuser www-data node modules/server-ce-scripts/scripts/check-redis.mjs + echo "All checks passed" +diff --git a/etc/my_init.d/900_run_web_migrations.sh b/etc/my_init.d/900_run_web_migrations.sh +index 59b7d23ea07f..cc206a528bb6 100755 +--- a/etc/my_init.d/900_run_web_migrations.sh ++++ b/etc/my_init.d/900_run_web_migrations.sh +@@ -9,5 +9,5 @@ fi + + echo "Running migrations for $environment" + cd /overleaf/services/web +-npm run migrations -- migrate -t "$environment" ++/sbin/setuser www-data npm run migrations -- migrate -t "$environment" + echo "Finished migrations" +diff --git a/etc/my_init.d/910_check_texlive_images b/etc/my_init.d/910_check_texlive_images +index 90dec0061f19..047dea5b603f 100755 +--- a/etc/my_init.d/910_check_texlive_images ++++ b/etc/my_init.d/910_check_texlive_images +@@ -3,4 +3,4 @@ set -e + + echo "Checking texlive images" + cd /overleaf/services/web +-node modules/server-ce-scripts/scripts/check-texlive-images.mjs ++/sbin/setuser www-data node modules/server-ce-scripts/scripts/check-texlive-images.mjs +diff --git a/etc/my_init.d/910_initiate_doc_version_recovery b/etc/my_init.d/910_initiate_doc_version_recovery +index 1daecd3c2fcd..0602e19872dc 100755 +--- a/etc/my_init.d/910_initiate_doc_version_recovery ++++ b/etc/my_init.d/910_initiate_doc_version_recovery +@@ -10,7 +10,7 @@ RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs- + + echo "Checking for doc version recovery. This can take a while if needed. Logs are in $LOG_FILE" + cd /overleaf/services/history-v1 +-LOG_LEVEL=info DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE="$RESYNCS_NEEDED_FILE" node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE" ++LOG_LEVEL=info DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE="$RESYNCS_NEEDED_FILE" /sbin/setuser www-data node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE" + + function resyncAllProjectsInBackground() { + waitForService docstore 3016 diff --git a/server-ce/hotfix/5.5.4/pr_27505.patch b/server-ce/hotfix/5.5.4/pr_27505.patch new file mode 100644 index 0000000000..50ed40d353 --- /dev/null +++ b/server-ce/hotfix/5.5.4/pr_27505.patch @@ -0,0 +1,250 @@ + + +diff --git a/services/history-v1/storage/lib/project_key.js b/services/history-v1/storage/lib/project_key.js +index 03fb2a5141ef..8727576b29f5 100644 +--- a/services/history-v1/storage/lib/project_key.js ++++ b/services/history-v1/storage/lib/project_key.js +@@ -1,5 +1,4 @@ + // Keep in sync with services/web/app/src/Features/History/project_key.js +-const _ = require('lodash') + const path = require('node:path') + + // +@@ -13,7 +12,7 @@ function format(projectId) { + } + + function pad(number) { +- return _.padStart(number, 9, '0') ++ return (number || 0).toString().padStart(9, '0') + } + + function naiveReverse(string) { +diff --git a/services/web/app/src/Features/History/project_key.js b/services/web/app/src/Features/History/project_key.js +index a4722db09afd..1630bdad0b2b 100644 +--- a/services/web/app/src/Features/History/project_key.js ++++ b/services/web/app/src/Features/History/project_key.js +@@ -1,5 +1,4 @@ + // Keep in sync with services/history-v1/storage/lib/project_key.js +-const _ = require('lodash') + const path = require('node:path') + + // +@@ -13,7 +12,7 @@ function format(projectId) { + } + + function pad(number) { +- return _.padStart(number, 9, '0') ++ return (number || 0).toString().padStart(9, '0') + } + + function naiveReverse(string) { + + +diff --git a/libraries/object-persistor/src/FSPersistor.js b/libraries/object-persistor/src/FSPersistor.js +index 38a81407df12..0b5891d2b2ed 100644 +--- a/libraries/object-persistor/src/FSPersistor.js ++++ b/libraries/object-persistor/src/FSPersistor.js +@@ -86,7 +86,7 @@ module.exports = class FSPersistor extends AbstractPersistor { + metric: 'fs.ingress', // ingress to us from disk + bucket: location, + }) +- const fsPath = this._getFsPath(location, name) ++ const fsPath = this._getFsPath(location, name, opts.useSubdirectories) + + try { + opts.fd = await fsPromises.open(fsPath, 'r') +@@ -295,9 +295,9 @@ module.exports = class FSPersistor extends AbstractPersistor { + await fsPromises.rm(dirPath, { force: true, recursive: true }) + } + +- _getFsPath(location, key) { ++ _getFsPath(location, key, useSubdirectories = false) { + key = key.replace(/\/$/, '') +- if (!this.useSubdirectories) { ++ if (!this.useSubdirectories && !useSubdirectories) { + key = key.replace(/\//g, '_') + } + return Path.join(location, key) +diff --git a/services/filestore/app.js b/services/filestore/app.js +index 24741e079c93..e69515ed7de0 100644 +--- a/services/filestore/app.js ++++ b/services/filestore/app.js +@@ -119,6 +119,17 @@ app.get( + fileController.getFile + ) + ++app.get( ++ '/history/global/hash/:hash', ++ keyBuilder.globalBlobFileKeyMiddleware, ++ fileController.getFile ++) ++app.get( ++ '/history/project/:historyId/hash/:hash', ++ keyBuilder.projectBlobFileKeyMiddleware, ++ fileController.getFile ++) ++ + app.get('/status', function (req, res) { + if (settings.shuttingDown) { + res.sendStatus(503) // Service unavailable +diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js +index 127bbcc20f83..2f77bd015da9 100644 +--- a/services/filestore/app/js/FileController.js ++++ b/services/filestore/app/js/FileController.js +@@ -25,6 +25,7 @@ function getFile(req, res, next) { + format, + style, + } ++ if (req.useSubdirectories) options.useSubdirectories = true + + metrics.inc('getFile') + req.requestLogger.setMessage('getting file') +diff --git a/services/filestore/app/js/KeyBuilder.js b/services/filestore/app/js/KeyBuilder.js +index f67a0e81d7ef..66c738171033 100644 +--- a/services/filestore/app/js/KeyBuilder.js ++++ b/services/filestore/app/js/KeyBuilder.js +@@ -1,4 +1,5 @@ + const settings = require('@overleaf/settings') ++const projectKey = require('./project_key') + + module.exports = { + getConvertedFolderKey, +@@ -6,6 +7,8 @@ module.exports = { + userFileKeyMiddleware, + userProjectKeyMiddleware, + bucketFileKeyMiddleware, ++ globalBlobFileKeyMiddleware, ++ projectBlobFileKeyMiddleware, + templateFileKeyMiddleware, + } + +@@ -50,6 +53,22 @@ function bucketFileKeyMiddleware(req, res, next) { + next() + } + ++function globalBlobFileKeyMiddleware(req, res, next) { ++ req.bucket = settings.filestore.stores.global_blobs ++ const { hash } = req.params ++ req.key = `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash.slice(4)}` ++ req.useSubdirectories = true ++ next() ++} ++ ++function projectBlobFileKeyMiddleware(req, res, next) { ++ req.bucket = settings.filestore.stores.project_blobs ++ const { historyId, hash } = req.params ++ req.key = `${projectKey.format(historyId)}/${hash.slice(0, 2)}/${hash.slice(2)}` ++ req.useSubdirectories = true ++ next() ++} ++ + function templateFileKeyMiddleware(req, res, next) { + const { + template_id: templateId, +diff --git a/services/web/app/src/Features/History/project_key.js b/services/filestore/app/js/project_key.js +similarity index 100% +rename from services/web/app/src/Features/History/project_key.js +rename to services/filestore/app/js/project_key.js +diff --git a/services/history-v1/storage/lib/project_key.js b/services/history-v1/storage/lib/project_key.js +index 8727576b29f5..6ad239dd12cb 100644 +--- a/services/history-v1/storage/lib/project_key.js ++++ b/services/history-v1/storage/lib/project_key.js +@@ -1,4 +1,4 @@ +-// Keep in sync with services/web/app/src/Features/History/project_key.js ++// Keep in sync with services/filestore/app/js/project_key.js + const path = require('node:path') + + // +diff --git a/services/web/app/src/Features/Compile/ClsiManager.js b/services/web/app/src/Features/Compile/ClsiManager.js +index 6f11297248ee..19370684dd80 100644 +--- a/services/web/app/src/Features/Compile/ClsiManager.js ++++ b/services/web/app/src/Features/Compile/ClsiManager.js +@@ -26,7 +26,7 @@ const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandle + const Metrics = require('@overleaf/metrics') + const Errors = require('../Errors/Errors') + const ClsiCacheHandler = require('./ClsiCacheHandler') +-const { getBlobLocation } = require('../History/HistoryManager') ++const { getFilestoreBlobURL } = require('../History/HistoryManager') + + const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex'] + const OUTPUT_FILE_TIMEOUT_MS = 60000 +@@ -755,8 +755,7 @@ function _finaliseRequest(projectId, options, project, docs, files) { + let url = filestoreURL + let fallbackURL + if (file.hash && Features.hasFeature('project-history-blobs')) { +- const { bucket, key } = getBlobLocation(historyId, file.hash) +- url = `${Settings.apis.filestore.url}/bucket/${bucket}/key/${key}` ++ url = getFilestoreBlobURL(historyId, file.hash) + fallbackURL = filestoreURL + } + resources.push({ +diff --git a/services/web/app/src/Features/History/HistoryManager.js b/services/web/app/src/Features/History/HistoryManager.js +index 42d7e229bf97..a2fb201399d1 100644 +--- a/services/web/app/src/Features/History/HistoryManager.js ++++ b/services/web/app/src/Features/History/HistoryManager.js +@@ -15,11 +15,6 @@ const { db, ObjectId, waitForDb } = require('../../infrastructure/mongodb') + const Metrics = require('@overleaf/metrics') + const logger = require('@overleaf/logger') + const { NotFoundError } = require('../Errors/Errors') +-const projectKey = require('./project_key') +- +-// BEGIN copy from services/history-v1/storage/lib/blob_store/index.js +- +-const GLOBAL_BLOBS = new Set() // CHANGE FROM SOURCE: only store hashes. + + const HISTORY_V1_URL = settings.apis.v1_history.url + const HISTORY_V1_BASIC_AUTH = { +@@ -27,27 +22,9 @@ const HISTORY_V1_BASIC_AUTH = { + password: settings.apis.v1_history.pass, + } + +-function makeGlobalKey(hash) { +- return `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash.slice(4)}` +-} +- +-function makeProjectKey(projectId, hash) { +- return `${projectKey.format(projectId)}/${hash.slice(0, 2)}/${hash.slice(2)}` +-} ++// BEGIN copy from services/history-v1/storage/lib/blob_store/index.js + +-function getBlobLocation(projectId, hash) { +- if (GLOBAL_BLOBS.has(hash)) { +- return { +- bucket: settings.apis.v1_history.buckets.globalBlobs, +- key: makeGlobalKey(hash), +- } +- } else { +- return { +- bucket: settings.apis.v1_history.buckets.projectBlobs, +- key: makeProjectKey(projectId, hash), +- } +- } +-} ++const GLOBAL_BLOBS = new Set() // CHANGE FROM SOURCE: only store hashes. + + async function loadGlobalBlobs() { + await waitForDb() // CHANGE FROM SOURCE: wait for db before running query. +@@ -59,6 +36,14 @@ async function loadGlobalBlobs() { + + // END copy from services/history-v1/storage/lib/blob_store/index.js + ++function getFilestoreBlobURL(historyId, hash) { ++ if (GLOBAL_BLOBS.has(hash)) { ++ return `${settings.apis.filestore.url}/history/global/hash/${hash}` ++ } else { ++ return `${settings.apis.filestore.url}/history/project/${historyId}/hash/${hash}` ++ } ++} ++ + async function initializeProject(projectId) { + const body = await fetchJson(`${settings.apis.project_history.url}/project`, { + method: 'POST', +@@ -421,7 +406,7 @@ function _userView(user) { + const loadGlobalBlobsPromise = loadGlobalBlobs() + + module.exports = { +- getBlobLocation, ++ getFilestoreBlobURL, + loadGlobalBlobsPromise, + initializeProject: callbackify(initializeProject), + flushProject: callbackify(flushProject), diff --git a/server-ce/hotfix/5.5.4/pr_27505.patch-abs b/server-ce/hotfix/5.5.4/pr_27505.patch-abs new file mode 100644 index 0000000000..4d06bf8670 --- /dev/null +++ b/server-ce/hotfix/5.5.4/pr_27505.patch-abs @@ -0,0 +1,29 @@ +diff --git a/etc/overleaf/settings.js b/etc/overleaf/settings.js +index 47d34fd8707e..c686a019f61f 100644 +--- a/etc/overleaf/settings.js ++++ b/etc/overleaf/settings.js +@@ -441,6 +441,8 @@ switch (process.env.OVERLEAF_FILESTORE_BACKEND) { + user_files: process.env.OVERLEAF_FILESTORE_USER_FILES_BUCKET_NAME, + template_files: + process.env.OVERLEAF_FILESTORE_TEMPLATE_FILES_BUCKET_NAME, ++ project_blobs: process.env.OVERLEAF_HISTORY_PROJECT_BLOBS_BUCKET, ++ global_blobs: process.env.OVERLEAF_HISTORY_BLOBS_BUCKET, + }, + s3: { + key: +@@ -463,6 +465,15 @@ switch (process.env.OVERLEAF_FILESTORE_BACKEND) { + stores: { + user_files: Path.join(DATA_DIR, 'user_files'), + template_files: Path.join(DATA_DIR, 'template_files'), ++ ++ // NOTE: The below paths are hard-coded in server-ce/config/production.json, so hard code them here as well. ++ // We can use DATA_DIR after switching history-v1 from 'config' to '@overleaf/settings'. ++ project_blobs: ++ process.env.OVERLEAF_HISTORY_PROJECT_BLOBS_BUCKET || ++ '/var/lib/overleaf/data/history/overleaf-project-blobs', ++ global_blobs: ++ process.env.OVERLEAF_HISTORY_BLOBS_BUCKET || ++ '/var/lib/overleaf/data/history/overleaf-global-blobs', + }, + } + }