agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-14 22:23:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #22208 from overleaf/jpa-clsi-hash

Browse Source 
[misc] clsi: read files from history-v1 with fallback to filestore

GitOrigin-RevId: c54bb128780198c14e7a63818f39fad62ce65d4e
This commit is contained in:
Jakob Ackermann
2024-11-29 09:05:39 +00:00
committed by Copybot
parent f6cbcc51d8
commit d19c5e236f
16 changed files with 431 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/web/app.mjs
+11 -2
View File
@@ -6,6 +6,7 @@ import metrics from '@overleaf/metrics'
import Settings from '@overleaf/settings'
import logger from '@overleaf/logger'
import PlansLocator from './app/src/Features/Subscription/PlansLocator.js'
import HistoryManager from './app/src/Features/History/HistoryManager.js'
import SiteAdminHandler from './app/src/infrastructure/SiteAdminHandler.js'
import http from 'node:http'
import https from 'node:https'
@@ -58,7 +59,11 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
PlansLocator.ensurePlansAreSetupCorrectly()
Promise.all([mongodb.connectionPromise, mongoose.connectionPromise])
Promise.all([
mongodb.connectionPromise,
mongoose.connectionPromise,
HistoryManager.promises.loadGlobalBlobs(),
])
.then(async () => {
Server.server.listen(port, host, function () {
logger.debug(`web starting up, listening on ${host}:${port}`)
@@ -76,7 +81,11 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
}
// initialise site admin tasks
Promise.all([mongodb.connectionPromise, mongoose.connectionPromise])
Promise.all([
mongodb.connectionPromise,
mongoose.connectionPromise,
HistoryManager.promises.loadGlobalBlobs(),
])
.then(() => SiteAdminHandler.initialise())
.catch(err => {
logger.fatal({ err }, 'Cannot connect to mongo. Exiting.')
services/web/app/src/Features/Compile/ClsiManager.js
+9 -1
View File
@@ -24,6 +24,7 @@ const ClsiFormatChecker = require('./ClsiFormatChecker')
const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
const Metrics = require('@overleaf/metrics')
const Errors = require('../Errors/Errors')
const { getBlobLocation } = require('../History/HistoryManager')
const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex']
const OUTPUT_FILE_TIMEOUT_MS = 60000
@@ -532,6 +533,7 @@ async function _buildRequest(projectId, options) {
rootDoc_id: 1,
imageName: 1,
rootFolder: 1,
'overleaf.history.id': 1,
})
if (project == null) {
throw new Errors.NotFoundError(`project does not exist: ${projectId}`)
@@ -731,12 +733,18 @@ function _finaliseRequest(projectId, options, project, docs, files) {
}
}
const historyId = project.overleaf.history.id
if (!historyId) {
throw new OError('project does not have a history id', { projectId })
}
for (let path in files) {
const file = files[path]
path = path.replace(/^\//, '') // Remove leading /
const { bucket, key } = getBlobLocation(historyId, file.hash)
resources.push({
path,
url: `${Settings.apis.filestore.url}/project/${project._id}/file/${file._id}`,
url: `${Settings.apis.filestore.url}/bucket/${bucket}/key/${key}`,
fallbackURL: `${Settings.apis.filestore.url}/project/${project._id}/file/${file._id}`,
modified: file.created?.getTime(),
})
}
services/web/app/src/Features/History/HistoryManager.js
+39 -1
View File
@@ -11,10 +11,46 @@ const OError = require('@overleaf/o-error')
const UserGetter = require('../User/UserGetter')
const ProjectGetter = require('../Project/ProjectGetter')
const HistoryBackupDeletionHandler = require('./HistoryBackupDeletionHandler')
const { ObjectId } = require('../../infrastructure/mongodb')
const { db, ObjectId } = require('../../infrastructure/mongodb')
const Metrics = require('@overleaf/metrics')
const logger = require('@overleaf/logger')
const { NotFoundError } = require('../Errors/Errors')
const projectKey = require('./project_key')
// BEGIN copy from services/history-v1/storage/lib/blob_store/index.js
const GLOBAL_BLOBS = new Set() // CHANGE FROM SOURCE: only store hashes.
function makeGlobalKey(hash) {
return `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash.slice(4)}`
}
function makeProjectKey(projectId, hash) {
return `${projectKey.format(projectId)}/${hash.slice(0, 2)}/${hash.slice(2)}`
}
function getBlobLocation(projectId, hash) {
if (GLOBAL_BLOBS.has(hash)) {
return {
bucket: settings.apis.v1_history.buckets.globalBlobs,
key: makeGlobalKey(hash),
}
} else {
return {
bucket: settings.apis.v1_history.buckets.projectBlobs,
key: makeProjectKey(projectId, hash),
}
}
}
async function loadGlobalBlobs() {
const blobs = db.projectHistoryGlobalBlobs.find()
for await (const blob of blobs) {
GLOBAL_BLOBS.add(blob._id) // CHANGE FROM SOURCE: only store hashes.
}
}
// END copy from services/history-v1/storage/lib/blob_store/index.js
async function initializeProject(projectId) {
const body = await fetchJson(`${settings.apis.project_history.url}/project`, {
@@ -357,6 +393,7 @@ function _userView(user) {
}
module.exports = {
getBlobLocation,
initializeProject: callbackify(initializeProject),
flushProject: callbackify(flushProject),
resyncProject: callbackify(resyncProject),
@@ -368,6 +405,7 @@ module.exports = {
copyBlob: callbackify(copyBlob),
requestBlobWithFallback: callbackify(requestBlobWithFallback),
promises: {
loadGlobalBlobs,
initializeProject,
flushProject,
resyncProject,
services/web/app/src/Features/History/project_key.js
+24
View File
@@ -0,0 +1,24 @@
// Keep in sync with services/history-v1/storage/lib/project_key.js
const _ = require('lodash')
const path = require('node:path')
//
// The advice in http://docs.aws.amazon.com/AmazonS3/latest/dev/
// request-rate-perf-considerations.html is to avoid sequential key prefixes,
// so we reverse the project ID part of the key as they suggest.
//
function format(projectId) {
const prefix = naiveReverse(pad(projectId))
return path.join(prefix.slice(0, 3), prefix.slice(3, 6), prefix.slice(6))
}
function pad(number) {
return _.padStart(number, 9, '0')
}
function naiveReverse(string) {
return string.split('').reverse().join('')
}
exports.format = format
exports.pad = pad
services/web/app/src/infrastructure/mongodb.js
+1
View File
@@ -68,6 +68,7 @@ const db = {
projectAuditLogEntries: internalDb.collection('projectAuditLogEntries'),
projectHistoryChunks: internalDb.collection('projectHistoryChunks'),
projectHistoryFailures: internalDb.collection('projectHistoryFailures'),
projectHistoryGlobalBlobs: internalDb.collection('projectHistoryGlobalBlobs'),
projectHistoryLabels: internalDb.collection('projectHistoryLabels'),
projectHistoryMetaData: internalDb.collection('projectHistoryMetaData'),
projectHistorySyncState: internalDb.collection('projectHistorySyncState'),
services/web/config/settings.defaults.js
+17
View File
@@ -275,6 +275,23 @@ module.exports = {
process.env.HAVE_I_BEEN_PWNED_URL || 'https://api.pwnedpasswords.com',
timeout: parseInt(process.env.HAVE_I_BEEN_PWNED_TIMEOUT, 10) || 5 * 1000,
},
v1_history: {
url:
process.env.V1_HISTORY_URL ||
`http://${process.env.V1_HISTORY_HOST || '127.0.0.1'}:${
process.env.V1_HISTORY_PORT || '3100'
}/api`,
user: process.env.V1_HISTORY_USER || 'staging',
pass:
process.env.V1_HISTORY_PASS ||
process.env.V1_HISTORY_PASSWORD ||
'password',
buckets: {
globalBlobs: process.env.OVERLEAF_EDITOR_BLOBS_BUCKET,
projectBlobs: process.env.OVERLEAF_EDITOR_PROJECT_BLOBS_BUCKET,
},
},
// For legacy reasons, we need to populate the below objects.
v1: {},
services/web/test/unit/src/Compile/ClsiManagerTests.js
+34 -3
View File
@@ -9,6 +9,8 @@ const FILESTORE_URL = 'http://filestore.example.com'
const CLSI_HOST = 'clsi.example.com'
const MODULE_PATH = '../../../../app/src/Features/Compile/ClsiManager.js'
const GLOBAL_BLOB_HASH = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
describe('ClsiManager', function () {
beforeEach(function () {
this.user_id = 'user-id'
@@ -17,6 +19,7 @@ describe('ClsiManager', function () {
compiler: 'latex',
rootDoc_id: 'mock-doc-id-1',
imageName: 'mock-image-name',
overleaf: { history: { id: 42 } },
}
this.docs = {
'/main.tex': {
@@ -31,10 +34,17 @@ describe('ClsiManager', function () {
},
}
this.files = {
'/images/image.png': {
name: 'image.png',
'/images/frog.png': {
name: 'frog.png',
_id: 'mock-file-id-1',
created: new Date(),
hash: GLOBAL_BLOB_HASH,
},
'/images/image.png': {
name: 'image.png',
_id: 'mock-file-id-2',
created: new Date(),
hash: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
},
}
this.clsiCookieKey = 'clsiserver'
@@ -129,6 +139,17 @@ describe('ClsiManager', function () {
enablePdfCaching: true,
clsiCookie: { key: 'clsiserver' },
}
this.HistoryManager = {
getBlobLocation: sinon.stub().callsFake((historyId, hash) => {
if (hash === GLOBAL_BLOB_HASH) {
return {
bucket: 'global-blobs',
key: 'aa/aa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
}
}
return { bucket: 'project-blobs', key: `${historyId}/${hash}` }
}),
}
this.ClsiManager = SandboxedModule.require(MODULE_PATH, {
requires: {
@@ -145,6 +166,7 @@ describe('ClsiManager', function () {
'@overleaf/fetch-utils': this.FetchUtils,
'./ClsiFormatChecker': this.ClsiFormatChecker,
'@overleaf/metrics': this.Metrics,
'../History/HistoryManager': this.HistoryManager,
},
})
tk.freeze(Date.now())
@@ -238,6 +260,7 @@ describe('ClsiManager', function () {
rootDoc_id: 1,
imageName: 1,
rootFolder: 1,
'overleaf.history.id': 1,
}
)
})
@@ -372,6 +395,7 @@ describe('ClsiManager', function () {
rootDoc_id: 1,
imageName: 1,
rootFolder: 1,
'overleaf.history.id': 1,
}
)
})
@@ -1003,9 +1027,16 @@ function _makeResources(project, docs, files) {
})
}
for (const [path, file] of Object.entries(files)) {
let url
if (file.hash === GLOBAL_BLOB_HASH) {
url = `${FILESTORE_URL}/bucket/global-blobs/key/aa/aa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`
} else {
url = `${FILESTORE_URL}/bucket/project-blobs/key/${project.overleaf.history.id}/${file.hash}`
}
resources.push({
path: path.replace(/^\//, ''),
url: `${FILESTORE_URL}/project/${project._id}/file/${file._id}`,
url,
fallbackURL: `${FILESTORE_URL}/project/${project._id}/file/${file._id}`,
modified: file.created.getTime(),
})
}