agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-05-23 17:19:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

[server-pro] add hotfix 5.5.4 (#27514)

Browse Source 
GitOrigin-RevId: f992fce918d2617017fa5af5be3edd9e54a5919d
This commit is contained in:
Jakob Ackermann
2025-07-30 17:54:48 +02:00
committed by Copybot
parent 29045b9085
commit 092c8aaf32
5 changed files with 440 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
server-ce/hotfix/5.5.4/Dockerfile Normal file
View File

@@ -0,0 +1,10 @@
FROM sharelatex/sharelatex:5.5.3
# ../../bin/import_pr_patch.sh 27504 27505
# Remove CE tests
# Remove web tests
# Fix cron paths
# Move init/shutdown/settings into .patch-abs file
COPY *.patch* .
RUN bash -ec 'for p in *.patch; do echo "=== Applying $p ==="; patch -p1 < "$p" && rm $p; done' \
&& bash -ec 'cd / && for p in /overleaf/*.patch-abs; do echo "=== Applying $p ==="; patch -p1 < "$p" && rm $p; done'

34
server-ce/hotfix/5.5.4/pr_27504.patch Normal file
View File

@@ -0,0 +1,34 @@
diff --git a/bin/flush-history-queues b/bin/flush-history-queues
index b54bc5558c6e..6c0cd896416e 100755
--- a/bin/flush-history-queues
+++ b/bin/flush-history-queues
@@ -5,4 +5,4 @@ set -euo pipefail
source /etc/container_environment.sh
source /etc/overleaf/env.sh
cd /overleaf/services/project-history
-node scripts/flush_all.js 100000
+exec /sbin/setuser www-data node scripts/flush_all.js 100000
diff --git a/bin/force-history-resyncs b/bin/force-history-resyncs
index 389c98a4ad69..4f48890855b7 100755
--- a/bin/force-history-resyncs
+++ b/bin/force-history-resyncs
@@ -5,4 +5,4 @@ set -euo pipefail
source /etc/container_environment.sh
source /etc/overleaf/env.sh
cd /overleaf/services/project-history
-node scripts/force_resync.js 1000 force
+exec /sbin/setuser www-data node scripts/force_resync.js 1000 force
diff --git a/cron/project-history-flush-all.sh b/cron/project-history-flush-all.sh
index 8fe9eea5fc55..689561737914 100755
--- a/cron/project-history-flush-all.sh
+++ b/cron/project-history-flush-all.sh
@@ -9,6 +9,6 @@ date
source /etc/container_environment.sh
source /etc/overleaf/env.sh
-cd /overleaf/services/project-history && node scripts/flush_all.js
+cd /overleaf/services/project-history && /sbin/setuser www-data node scripts/flush_all.js
echo "Done flushing all project-history changes"

117
server-ce/hotfix/5.5.4/pr_27504.patch-abs Normal file
View File

@@ -0,0 +1,117 @@
diff --git a/usr/local/bin/grunt b/usr/local/bin/grunt
index 462c68df4d52..8595d67109ae 100755
--- a/usr/local/bin/grunt
+++ b/usr/local/bin/grunt
@@ -11,22 +11,22 @@ cd /overleaf/services/web
case "$TASK" in
user:create-admin)
echo "The grunt command is deprecated, run the create-user script using node instead"
- node modules/server-ce-scripts/scripts/create-user.mjs --admin "$@"
+ exec /sbin/setuser www-data node modules/server-ce-scripts/scripts/create-user.mjs --admin "$@"
;;
user:delete)
echo "The grunt command is deprecated, run the delete-user script using node instead"
- node modules/server-ce-scripts/scripts/delete-user.mjs "$@"
+ exec /sbin/setuser www-data node modules/server-ce-scripts/scripts/delete-user.mjs "$@"
;;
check:mongo)
echo "The grunt command is deprecated, run the check-mongodb script using node instead"
- node modules/server-ce-scripts/scripts/check-mongodb.mjs
+ exec /sbin/setuser www-data node modules/server-ce-scripts/scripts/check-mongodb.mjs
;;
check:redis)
echo "The grunt command is deprecated, run the check-redis script using node instead"
- node modules/server-ce-scripts/scripts/check-redis.mjs
+ exec /sbin/setuser www-data node modules/server-ce-scripts/scripts/check-redis.mjs
;;
*)
diff --git a/etc/my_init.pre_shutdown.d/00_close_site b/etc/my_init.pre_shutdown.d/00_close_site
index ed5404f8172f..ac579f4b10ff 100755
--- a/etc/my_init.pre_shutdown.d/00_close_site
+++ b/etc/my_init.pre_shutdown.d/00_close_site
@@ -12,7 +12,7 @@ echo "closed" > "${SITE_MAINTENANCE_FILE}"
sleep 5
# giving a grace period of 5 seconds for users before disconnecting them and start shutting down
-cd /overleaf/services/web && node scripts/disconnect_all_users.mjs --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1
+cd /overleaf/services/web && /sbin/setuser www-data node scripts/disconnect_all_users.mjs --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1
EXIT_CODE="$?"
if [ $EXIT_CODE -ne 0 ]
diff --git a/etc/my_init.pre_shutdown.d/01_flush_document_updater b/etc/my_init.pre_shutdown.d/01_flush_document_updater
index 0900fe5fac04..b4529f856b1e 100755
--- a/etc/my_init.pre_shutdown.d/01_flush_document_updater
+++ b/etc/my_init.pre_shutdown.d/01_flush_document_updater
@@ -3,7 +3,7 @@
. /etc/container_environment.sh
. /etc/overleaf/env.sh
-cd /overleaf/services/document-updater && node scripts/flush_all.js >> /var/log/overleaf/document-updater.log 2>&1
+cd /overleaf/services/document-updater && /sbin/setuser www-data node scripts/flush_all.js >> /var/log/overleaf/document-updater.log 2>&1
EXIT_CODE="$?"
if [ $EXIT_CODE -ne 0 ]
diff --git a/etc/my_init.pre_shutdown.d/02_flush_project_history b/etc/my_init.pre_shutdown.d/02_flush_project_history
index f8ac51600c98..2844379cebc1 100755
--- a/etc/my_init.pre_shutdown.d/02_flush_project_history
+++ b/etc/my_init.pre_shutdown.d/02_flush_project_history
@@ -3,7 +3,7 @@
. /etc/container_environment.sh
. /etc/overleaf/env.sh
-cd /overleaf/services/project-history && node scripts/flush_all.js >> /var/log/overleaf/project-history.log 2>&1
+cd /overleaf/services/project-history && /sbin/setuser www-data node scripts/flush_all.js >> /var/log/overleaf/project-history.log 2>&1
EXIT_CODE="$?"
if [ $EXIT_CODE -ne 0 ]
diff --git a/etc/my_init.d/500_check_db_access.sh b/etc/my_init.d/500_check_db_access.sh
index bbf2b9ec267d..f71acc8e011e 100755
--- a/etc/my_init.d/500_check_db_access.sh
+++ b/etc/my_init.d/500_check_db_access.sh
@@ -3,6 +3,6 @@ set -e
echo "Checking can connect to mongo and redis"
cd /overleaf/services/web
-node modules/server-ce-scripts/scripts/check-mongodb.mjs
-node modules/server-ce-scripts/scripts/check-redis.mjs
+/sbin/setuser www-data node modules/server-ce-scripts/scripts/check-mongodb.mjs
+/sbin/setuser www-data node modules/server-ce-scripts/scripts/check-redis.mjs
echo "All checks passed"
diff --git a/etc/my_init.d/900_run_web_migrations.sh b/etc/my_init.d/900_run_web_migrations.sh
index 59b7d23ea07f..cc206a528bb6 100755
--- a/etc/my_init.d/900_run_web_migrations.sh
+++ b/etc/my_init.d/900_run_web_migrations.sh
@@ -9,5 +9,5 @@ fi
echo "Running migrations for $environment"
cd /overleaf/services/web
-npm run migrations -- migrate -t "$environment"
+/sbin/setuser www-data npm run migrations -- migrate -t "$environment"
echo "Finished migrations"
diff --git a/etc/my_init.d/910_check_texlive_images b/etc/my_init.d/910_check_texlive_images
index 90dec0061f19..047dea5b603f 100755
--- a/etc/my_init.d/910_check_texlive_images
+++ b/etc/my_init.d/910_check_texlive_images
@@ -3,4 +3,4 @@ set -e
echo "Checking texlive images"
cd /overleaf/services/web
-node modules/server-ce-scripts/scripts/check-texlive-images.mjs
+/sbin/setuser www-data node modules/server-ce-scripts/scripts/check-texlive-images.mjs
diff --git a/etc/my_init.d/910_initiate_doc_version_recovery b/etc/my_init.d/910_initiate_doc_version_recovery
index 1daecd3c2fcd..0602e19872dc 100755
--- a/etc/my_init.d/910_initiate_doc_version_recovery
+++ b/etc/my_init.d/910_initiate_doc_version_recovery
@@ -10,7 +10,7 @@ RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs-
echo "Checking for doc version recovery. This can take a while if needed. Logs are in $LOG_FILE"
cd /overleaf/services/history-v1
-LOG_LEVEL=info DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE="$RESYNCS_NEEDED_FILE" node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE"
+LOG_LEVEL=info DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE="$RESYNCS_NEEDED_FILE" /sbin/setuser www-data node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE"
function resyncAllProjectsInBackground() {
waitForService docstore 3016

250
server-ce/hotfix/5.5.4/pr_27505.patch Normal file
View File

@@ -0,0 +1,250 @@
diff --git a/services/history-v1/storage/lib/project_key.js b/services/history-v1/storage/lib/project_key.js
index 03fb2a5141ef..8727576b29f5 100644
--- a/services/history-v1/storage/lib/project_key.js
+++ b/services/history-v1/storage/lib/project_key.js
@@ -1,5 +1,4 @@
// Keep in sync with services/web/app/src/Features/History/project_key.js
-const _ = require('lodash')
const path = require('node:path')
//
@@ -13,7 +12,7 @@ function format(projectId) {
}
function pad(number) {
- return _.padStart(number, 9, '0')
+ return (number || 0).toString().padStart(9, '0')
}
function naiveReverse(string) {
diff --git a/services/web/app/src/Features/History/project_key.js b/services/web/app/src/Features/History/project_key.js
index a4722db09afd..1630bdad0b2b 100644
--- a/services/web/app/src/Features/History/project_key.js
+++ b/services/web/app/src/Features/History/project_key.js
@@ -1,5 +1,4 @@
// Keep in sync with services/history-v1/storage/lib/project_key.js
-const _ = require('lodash')
const path = require('node:path')
//
@@ -13,7 +12,7 @@ function format(projectId) {
}
function pad(number) {
- return _.padStart(number, 9, '0')
+ return (number || 0).toString().padStart(9, '0')
}
function naiveReverse(string) {
diff --git a/libraries/object-persistor/src/FSPersistor.js b/libraries/object-persistor/src/FSPersistor.js
index 38a81407df12..0b5891d2b2ed 100644
--- a/libraries/object-persistor/src/FSPersistor.js
+++ b/libraries/object-persistor/src/FSPersistor.js
@@ -86,7 +86,7 @@ module.exports = class FSPersistor extends AbstractPersistor {
metric: 'fs.ingress', // ingress to us from disk
bucket: location,
})
- const fsPath = this._getFsPath(location, name)
+ const fsPath = this._getFsPath(location, name, opts.useSubdirectories)
try {
opts.fd = await fsPromises.open(fsPath, 'r')
@@ -295,9 +295,9 @@ module.exports = class FSPersistor extends AbstractPersistor {
await fsPromises.rm(dirPath, { force: true, recursive: true })
}
- _getFsPath(location, key) {
+ _getFsPath(location, key, useSubdirectories = false) {
key = key.replace(/\/$/, '')
- if (!this.useSubdirectories) {
+ if (!this.useSubdirectories && !useSubdirectories) {
key = key.replace(/\//g, '_')
}
return Path.join(location, key)
diff --git a/services/filestore/app.js b/services/filestore/app.js
index 24741e079c93..e69515ed7de0 100644
--- a/services/filestore/app.js
+++ b/services/filestore/app.js
@@ -119,6 +119,17 @@ app.get(
fileController.getFile
)
+app.get(
+ '/history/global/hash/:hash',
+ keyBuilder.globalBlobFileKeyMiddleware,
+ fileController.getFile
+)
+app.get(
+ '/history/project/:historyId/hash/:hash',
+ keyBuilder.projectBlobFileKeyMiddleware,
+ fileController.getFile
+)
+
app.get('/status', function (req, res) {
if (settings.shuttingDown) {
res.sendStatus(503) // Service unavailable
diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js
index 127bbcc20f83..2f77bd015da9 100644
--- a/services/filestore/app/js/FileController.js
+++ b/services/filestore/app/js/FileController.js
@@ -25,6 +25,7 @@ function getFile(req, res, next) {
format,
style,
}
+ if (req.useSubdirectories) options.useSubdirectories = true
metrics.inc('getFile')
req.requestLogger.setMessage('getting file')
diff --git a/services/filestore/app/js/KeyBuilder.js b/services/filestore/app/js/KeyBuilder.js
index f67a0e81d7ef..66c738171033 100644
--- a/services/filestore/app/js/KeyBuilder.js
+++ b/services/filestore/app/js/KeyBuilder.js
@@ -1,4 +1,5 @@
const settings = require('@overleaf/settings')
+const projectKey = require('./project_key')
module.exports = {
getConvertedFolderKey,
@@ -6,6 +7,8 @@ module.exports = {
userFileKeyMiddleware,
userProjectKeyMiddleware,
bucketFileKeyMiddleware,
+ globalBlobFileKeyMiddleware,
+ projectBlobFileKeyMiddleware,
templateFileKeyMiddleware,
}
@@ -50,6 +53,22 @@ function bucketFileKeyMiddleware(req, res, next) {
next()
}
+function globalBlobFileKeyMiddleware(req, res, next) {
+ req.bucket = settings.filestore.stores.global_blobs
+ const { hash } = req.params
+ req.key = `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash.slice(4)}`
+ req.useSubdirectories = true
+ next()
+}
+
+function projectBlobFileKeyMiddleware(req, res, next) {
+ req.bucket = settings.filestore.stores.project_blobs
+ const { historyId, hash } = req.params
+ req.key = `${projectKey.format(historyId)}/${hash.slice(0, 2)}/${hash.slice(2)}`
+ req.useSubdirectories = true
+ next()
+}
+
function templateFileKeyMiddleware(req, res, next) {
const {
template_id: templateId,
diff --git a/services/web/app/src/Features/History/project_key.js b/services/filestore/app/js/project_key.js
similarity index 100%
rename from services/web/app/src/Features/History/project_key.js
rename to services/filestore/app/js/project_key.js
diff --git a/services/history-v1/storage/lib/project_key.js b/services/history-v1/storage/lib/project_key.js
index 8727576b29f5..6ad239dd12cb 100644
--- a/services/history-v1/storage/lib/project_key.js
+++ b/services/history-v1/storage/lib/project_key.js
@@ -1,4 +1,4 @@
-// Keep in sync with services/web/app/src/Features/History/project_key.js
+// Keep in sync with services/filestore/app/js/project_key.js
const path = require('node:path')
//
diff --git a/services/web/app/src/Features/Compile/ClsiManager.js b/services/web/app/src/Features/Compile/ClsiManager.js
index 6f11297248ee..19370684dd80 100644
--- a/services/web/app/src/Features/Compile/ClsiManager.js
+++ b/services/web/app/src/Features/Compile/ClsiManager.js
@@ -26,7 +26,7 @@ const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandle
const Metrics = require('@overleaf/metrics')
const Errors = require('../Errors/Errors')
const ClsiCacheHandler = require('./ClsiCacheHandler')
-const { getBlobLocation } = require('../History/HistoryManager')
+const { getFilestoreBlobURL } = require('../History/HistoryManager')
const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex']
const OUTPUT_FILE_TIMEOUT_MS = 60000
@@ -755,8 +755,7 @@ function _finaliseRequest(projectId, options, project, docs, files) {
let url = filestoreURL
let fallbackURL
if (file.hash && Features.hasFeature('project-history-blobs')) {
- const { bucket, key } = getBlobLocation(historyId, file.hash)
- url = `${Settings.apis.filestore.url}/bucket/${bucket}/key/${key}`
+ url = getFilestoreBlobURL(historyId, file.hash)
fallbackURL = filestoreURL
}
resources.push({
diff --git a/services/web/app/src/Features/History/HistoryManager.js b/services/web/app/src/Features/History/HistoryManager.js
index 42d7e229bf97..a2fb201399d1 100644
--- a/services/web/app/src/Features/History/HistoryManager.js
+++ b/services/web/app/src/Features/History/HistoryManager.js
@@ -15,11 +15,6 @@ const { db, ObjectId, waitForDb } = require('../../infrastructure/mongodb')
const Metrics = require('@overleaf/metrics')
const logger = require('@overleaf/logger')
const { NotFoundError } = require('../Errors/Errors')
-const projectKey = require('./project_key')
-
-// BEGIN copy from services/history-v1/storage/lib/blob_store/index.js
-
-const GLOBAL_BLOBS = new Set() // CHANGE FROM SOURCE: only store hashes.
const HISTORY_V1_URL = settings.apis.v1_history.url
const HISTORY_V1_BASIC_AUTH = {
@@ -27,27 +22,9 @@ const HISTORY_V1_BASIC_AUTH = {
password: settings.apis.v1_history.pass,
}
-function makeGlobalKey(hash) {
- return `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash.slice(4)}`
-}
-
-function makeProjectKey(projectId, hash) {
- return `${projectKey.format(projectId)}/${hash.slice(0, 2)}/${hash.slice(2)}`
-}
+// BEGIN copy from services/history-v1/storage/lib/blob_store/index.js
-function getBlobLocation(projectId, hash) {
- if (GLOBAL_BLOBS.has(hash)) {
- return {
- bucket: settings.apis.v1_history.buckets.globalBlobs,
- key: makeGlobalKey(hash),
- }
- } else {
- return {
- bucket: settings.apis.v1_history.buckets.projectBlobs,
- key: makeProjectKey(projectId, hash),
- }
- }
-}
+const GLOBAL_BLOBS = new Set() // CHANGE FROM SOURCE: only store hashes.
async function loadGlobalBlobs() {
await waitForDb() // CHANGE FROM SOURCE: wait for db before running query.
@@ -59,6 +36,14 @@ async function loadGlobalBlobs() {
// END copy from services/history-v1/storage/lib/blob_store/index.js
+function getFilestoreBlobURL(historyId, hash) {
+ if (GLOBAL_BLOBS.has(hash)) {
+ return `${settings.apis.filestore.url}/history/global/hash/${hash}`
+ } else {
+ return `${settings.apis.filestore.url}/history/project/${historyId}/hash/${hash}`
+ }
+}
+
async function initializeProject(projectId) {
const body = await fetchJson(`${settings.apis.project_history.url}/project`, {
method: 'POST',
@@ -421,7 +406,7 @@ function _userView(user) {
const loadGlobalBlobsPromise = loadGlobalBlobs()
module.exports = {
- getBlobLocation,
+ getFilestoreBlobURL,
loadGlobalBlobsPromise,
initializeProject: callbackify(initializeProject),
flushProject: callbackify(flushProject),

29
server-ce/hotfix/5.5.4/pr_27505.patch-abs Normal file
View File

@@ -0,0 +1,29 @@
diff --git a/etc/overleaf/settings.js b/etc/overleaf/settings.js
index 47d34fd8707e..c686a019f61f 100644
--- a/etc/overleaf/settings.js
+++ b/etc/overleaf/settings.js
@@ -441,6 +441,8 @@ switch (process.env.OVERLEAF_FILESTORE_BACKEND) {
user_files: process.env.OVERLEAF_FILESTORE_USER_FILES_BUCKET_NAME,
template_files:
process.env.OVERLEAF_FILESTORE_TEMPLATE_FILES_BUCKET_NAME,
+ project_blobs: process.env.OVERLEAF_HISTORY_PROJECT_BLOBS_BUCKET,
+ global_blobs: process.env.OVERLEAF_HISTORY_BLOBS_BUCKET,
},
s3: {
key:
@@ -463,6 +465,15 @@ switch (process.env.OVERLEAF_FILESTORE_BACKEND) {
stores: {
user_files: Path.join(DATA_DIR, 'user_files'),
template_files: Path.join(DATA_DIR, 'template_files'),
+
+ // NOTE: The below paths are hard-coded in server-ce/config/production.json, so hard code them here as well.
+ // We can use DATA_DIR after switching history-v1 from 'config' to '@overleaf/settings'.
+ project_blobs:
+ process.env.OVERLEAF_HISTORY_PROJECT_BLOBS_BUCKET ||
+ '/var/lib/overleaf/data/history/overleaf-project-blobs',
+ global_blobs:
+ process.env.OVERLEAF_HISTORY_BLOBS_BUCKET ||
+ '/var/lib/overleaf/data/history/overleaf-global-blobs',
},
}
}