diff --git a/services/history-v1/storage/lib/project_key.js b/services/history-v1/storage/lib/project_key.js index 03fb2a5141ef..8727576b29f5 100644 --- a/services/history-v1/storage/lib/project_key.js +++ b/services/history-v1/storage/lib/project_key.js @@ -1,5 +1,4 @@ // Keep in sync with services/web/app/src/Features/History/project_key.js -const _ = require('lodash') const path = require('node:path') // @@ -13,7 +12,7 @@ function format(projectId) { } function pad(number) { - return _.padStart(number, 9, '0') + return (number || 0).toString().padStart(9, '0') } function naiveReverse(string) { diff --git a/services/web/app/src/Features/History/project_key.js b/services/web/app/src/Features/History/project_key.js index a4722db09afd..1630bdad0b2b 100644 --- a/services/web/app/src/Features/History/project_key.js +++ b/services/web/app/src/Features/History/project_key.js @@ -1,5 +1,4 @@ // Keep in sync with services/history-v1/storage/lib/project_key.js -const _ = require('lodash') const path = require('node:path') // @@ -13,7 +12,7 @@ function format(projectId) { } function pad(number) { - return _.padStart(number, 9, '0') + return (number || 0).toString().padStart(9, '0') } function naiveReverse(string) { diff --git a/libraries/object-persistor/src/FSPersistor.js b/libraries/object-persistor/src/FSPersistor.js index 38a81407df12..0b5891d2b2ed 100644 --- a/libraries/object-persistor/src/FSPersistor.js +++ b/libraries/object-persistor/src/FSPersistor.js @@ -86,7 +86,7 @@ module.exports = class FSPersistor extends AbstractPersistor { metric: 'fs.ingress', // ingress to us from disk bucket: location, }) - const fsPath = this._getFsPath(location, name) + const fsPath = this._getFsPath(location, name, opts.useSubdirectories) try { opts.fd = await fsPromises.open(fsPath, 'r') @@ -295,9 +295,9 @@ module.exports = class FSPersistor extends AbstractPersistor { await fsPromises.rm(dirPath, { force: true, recursive: true }) } - _getFsPath(location, key) { + _getFsPath(location, key, useSubdirectories = false) { key = key.replace(/\/$/, '') - if (!this.useSubdirectories) { + if (!this.useSubdirectories && !useSubdirectories) { key = key.replace(/\//g, '_') } return Path.join(location, key) diff --git a/services/filestore/app.js b/services/filestore/app.js index 24741e079c93..e69515ed7de0 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -119,6 +119,17 @@ app.get( fileController.getFile ) +app.get( + '/history/global/hash/:hash', + keyBuilder.globalBlobFileKeyMiddleware, + fileController.getFile +) +app.get( + '/history/project/:historyId/hash/:hash', + keyBuilder.projectBlobFileKeyMiddleware, + fileController.getFile +) + app.get('/status', function (req, res) { if (settings.shuttingDown) { res.sendStatus(503) // Service unavailable diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js index 127bbcc20f83..2f77bd015da9 100644 --- a/services/filestore/app/js/FileController.js +++ b/services/filestore/app/js/FileController.js @@ -25,6 +25,7 @@ function getFile(req, res, next) { format, style, } + if (req.useSubdirectories) options.useSubdirectories = true metrics.inc('getFile') req.requestLogger.setMessage('getting file') diff --git a/services/filestore/app/js/KeyBuilder.js b/services/filestore/app/js/KeyBuilder.js index f67a0e81d7ef..66c738171033 100644 --- a/services/filestore/app/js/KeyBuilder.js +++ b/services/filestore/app/js/KeyBuilder.js @@ -1,4 +1,5 @@ const settings = require('@overleaf/settings') +const projectKey = require('./project_key') module.exports = { getConvertedFolderKey, @@ -6,6 +7,8 @@ module.exports = { userFileKeyMiddleware, userProjectKeyMiddleware, bucketFileKeyMiddleware, + globalBlobFileKeyMiddleware, + projectBlobFileKeyMiddleware, templateFileKeyMiddleware, } @@ -50,6 +53,22 @@ function bucketFileKeyMiddleware(req, res, next) { next() } +function globalBlobFileKeyMiddleware(req, res, next) { + req.bucket = settings.filestore.stores.global_blobs + const { hash } = req.params + req.key = `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash.slice(4)}` + req.useSubdirectories = true + next() +} + +function projectBlobFileKeyMiddleware(req, res, next) { + req.bucket = settings.filestore.stores.project_blobs + const { historyId, hash } = req.params + req.key = `${projectKey.format(historyId)}/${hash.slice(0, 2)}/${hash.slice(2)}` + req.useSubdirectories = true + next() +} + function templateFileKeyMiddleware(req, res, next) { const { template_id: templateId, diff --git a/services/web/app/src/Features/History/project_key.js b/services/filestore/app/js/project_key.js similarity index 100% rename from services/web/app/src/Features/History/project_key.js rename to services/filestore/app/js/project_key.js diff --git a/services/history-v1/storage/lib/project_key.js b/services/history-v1/storage/lib/project_key.js index 8727576b29f5..6ad239dd12cb 100644 --- a/services/history-v1/storage/lib/project_key.js +++ b/services/history-v1/storage/lib/project_key.js @@ -1,4 +1,4 @@ -// Keep in sync with services/web/app/src/Features/History/project_key.js +// Keep in sync with services/filestore/app/js/project_key.js const path = require('node:path') // diff --git a/services/web/app/src/Features/Compile/ClsiManager.js b/services/web/app/src/Features/Compile/ClsiManager.js index 6f11297248ee..19370684dd80 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.js +++ b/services/web/app/src/Features/Compile/ClsiManager.js @@ -26,7 +26,7 @@ const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandle const Metrics = require('@overleaf/metrics') const Errors = require('../Errors/Errors') const ClsiCacheHandler = require('./ClsiCacheHandler') -const { getBlobLocation } = require('../History/HistoryManager') +const { getFilestoreBlobURL } = require('../History/HistoryManager') const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex'] const OUTPUT_FILE_TIMEOUT_MS = 60000 @@ -755,8 +755,7 @@ function _finaliseRequest(projectId, options, project, docs, files) { let url = filestoreURL let fallbackURL if (file.hash && Features.hasFeature('project-history-blobs')) { - const { bucket, key } = getBlobLocation(historyId, file.hash) - url = `${Settings.apis.filestore.url}/bucket/${bucket}/key/${key}` + url = getFilestoreBlobURL(historyId, file.hash) fallbackURL = filestoreURL } resources.push({ diff --git a/services/web/app/src/Features/History/HistoryManager.js b/services/web/app/src/Features/History/HistoryManager.js index 42d7e229bf97..a2fb201399d1 100644 --- a/services/web/app/src/Features/History/HistoryManager.js +++ b/services/web/app/src/Features/History/HistoryManager.js @@ -15,11 +15,6 @@ const { db, ObjectId, waitForDb } = require('../../infrastructure/mongodb') const Metrics = require('@overleaf/metrics') const logger = require('@overleaf/logger') const { NotFoundError } = require('../Errors/Errors') -const projectKey = require('./project_key') - -// BEGIN copy from services/history-v1/storage/lib/blob_store/index.js - -const GLOBAL_BLOBS = new Set() // CHANGE FROM SOURCE: only store hashes. const HISTORY_V1_URL = settings.apis.v1_history.url const HISTORY_V1_BASIC_AUTH = { @@ -27,27 +22,9 @@ const HISTORY_V1_BASIC_AUTH = { password: settings.apis.v1_history.pass, } -function makeGlobalKey(hash) { - return `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash.slice(4)}` -} - -function makeProjectKey(projectId, hash) { - return `${projectKey.format(projectId)}/${hash.slice(0, 2)}/${hash.slice(2)}` -} +// BEGIN copy from services/history-v1/storage/lib/blob_store/index.js -function getBlobLocation(projectId, hash) { - if (GLOBAL_BLOBS.has(hash)) { - return { - bucket: settings.apis.v1_history.buckets.globalBlobs, - key: makeGlobalKey(hash), - } - } else { - return { - bucket: settings.apis.v1_history.buckets.projectBlobs, - key: makeProjectKey(projectId, hash), - } - } -} +const GLOBAL_BLOBS = new Set() // CHANGE FROM SOURCE: only store hashes. async function loadGlobalBlobs() { await waitForDb() // CHANGE FROM SOURCE: wait for db before running query. @@ -59,6 +36,14 @@ async function loadGlobalBlobs() { // END copy from services/history-v1/storage/lib/blob_store/index.js +function getFilestoreBlobURL(historyId, hash) { + if (GLOBAL_BLOBS.has(hash)) { + return `${settings.apis.filestore.url}/history/global/hash/${hash}` + } else { + return `${settings.apis.filestore.url}/history/project/${historyId}/hash/${hash}` + } +} + async function initializeProject(projectId) { const body = await fetchJson(`${settings.apis.project_history.url}/project`, { method: 'POST', @@ -421,7 +406,7 @@ function _userView(user) { const loadGlobalBlobsPromise = loadGlobalBlobs() module.exports = { - getBlobLocation, + getFilestoreBlobURL, loadGlobalBlobsPromise, initializeProject: callbackify(initializeProject), flushProject: callbackify(flushProject),