agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-14 14:13:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #22533 from overleaf/ar-only-use-history-for-blobs-when-enabled

Browse Source 
[web] only use history for blobs when enabled

GitOrigin-RevId: 010983e9b29657d4c594e03945dca5700577bf0a
This commit is contained in:
Brian Gough
2025-01-10 09:05:32 +00:00
committed by Copybot
parent eb1b939f5a
commit 2ce82fabab
22 changed files with 303 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/web/app/src/Features/Compile/ClsiManager.js
+2 -1
View File
@@ -15,6 +15,7 @@ const { Cookie } = require('tough-cookie')
const ClsiCookieManager = require('./ClsiCookieManager')(
Settings.apis.clsi?.backendGroupName
)
const Features = require('../../infrastructure/Features')
const NewBackendCloudClsiCookieManager = require('./ClsiCookieManager')(
Settings.apis.clsi_new?.backendGroupName
)
@@ -744,7 +745,7 @@ function _finaliseRequest(projectId, options, project, docs, files) {
const filestoreURL = `${Settings.apis.filestore.url}/project/${project._id}/file/${file._id}`
let url = filestoreURL
let fallbackURL
if (file.hash) {
if (file.hash && Features.hasFeature('project-history-blobs')) {
const { bucket, key } = getBlobLocation(historyId, file.hash)
url = `${Settings.apis.filestore.url}/bucket/${bucket}/key/${key}`
fallbackURL = filestoreURL
services/web/app/src/Features/Downloads/ProjectZipStreamManager.mjs
+24 -5
View File
@@ -4,6 +4,8 @@ import logger from '@overleaf/logger'
import ProjectEntityHandler from '../Project/ProjectEntityHandler.js'
import ProjectGetter from '../Project/ProjectGetter.js'
import HistoryManager from '../History/HistoryManager.js'
import FileStoreHandler from '../FileStore/FileStoreHandler.js'
import Features from '../../infrastructure/Features.js'
let ProjectZipStreamManager
export default ProjectZipStreamManager = {
@@ -108,17 +110,35 @@ export default ProjectZipStreamManager = {
})
},
getFileStream: (projectId, file, callback) => {
if (Features.hasFeature('project-history-blobs')) {
HistoryManager.requestBlobWithFallback(
projectId,
file.hash,
file._id,
(error, result) => {
if (error) {
return callback(error)
}
const { stream } = result
callback(null, stream)
}
)
} else {
FileStoreHandler.getFileStream(projectId, file._id, {}, callback)
}
},
addAllFilesToArchive(projectId, archive, callback) {
ProjectEntityHandler.getAllFiles(projectId, (error, files) => {
if (error) {
return callback(error)
}
const jobs = Object.entries(files).map(([path, file]) => cb => {
HistoryManager.requestBlobWithFallback(
ProjectZipStreamManager.getFileStream(
projectId,
file.hash,
file._id,
(error, result) => {
file,
(error, stream) => {
if (error) {
logger.warn(
{ err: error, projectId, fileId: file._id },
@@ -129,7 +149,6 @@ export default ProjectZipStreamManager = {
if (path[0] === '/') {
path = path.slice(1)
}
const { stream } = result
archive.append(stream, { name: path })
stream.on('end', () => cb())
}
services/web/app/src/Features/FileStore/FileStoreHandler.js
+35 -22
View File
@@ -11,6 +11,7 @@ const { File } = require('../../models/File')
const Errors = require('../Errors/Errors')
const OError = require('@overleaf/o-error')
const { promisifyAll } = require('@overleaf/promise-utils')
const Features = require('../../infrastructure/Features')
const ONE_MIN_IN_MS = 60 * 1000
const FIVE_MINS_IN_MS = ONE_MIN_IN_MS * 5
@@ -38,6 +39,28 @@ const FileStoreHandler = {
})
},
_uploadToHistory(historyId, hash, size, fsPath, callback) {
if (Features.hasFeature('project-history-blobs')) {
Async.retry(
FileStoreHandler.RETRY_ATTEMPTS,
cb =>
HistoryManager.uploadBlobFromDisk(historyId, hash, size, fsPath, cb),
error => callback(error, true)
)
} else {
callback(null, false)
}
},
_uploadToFileStore(projectId, fileArgs, fsPath, callback) {
Async.retry(
FileStoreHandler.RETRY_ATTEMPTS,
(cb, results) =>
FileStoreHandler._doUploadFileFromDisk(projectId, fileArgs, fsPath, cb),
callback
)
},
uploadFileFromDiskWithHistoryId(
projectId,
historyId,
@@ -68,30 +91,20 @@ const FileStoreHandler = {
if (err) {
return callback(err)
}
Async.retry(
FileStoreHandler.RETRY_ATTEMPTS,
cb =>
HistoryManager.uploadBlobFromDisk(
historyId,
hash,
stat.size,
fsPath,
cb
),
function (err) {
FileStoreHandler._uploadToHistory(
historyId,
hash,
stat.size,
fsPath,
function (err, createdBlob) {
if (err) {
return callback(err)
}
fileArgs = { ...fileArgs, hash }
Async.retry(
FileStoreHandler.RETRY_ATTEMPTS,
(cb, results) =>
FileStoreHandler._doUploadFileFromDisk(
projectId,
fileArgs,
fsPath,
cb
),
FileStoreHandler._uploadToFileStore(
projectId,
fileArgs,
fsPath,
function (err, result) {
if (err) {
OError.tag(err, 'Error uploading file, retries failed', {
@@ -100,7 +113,7 @@ const FileStoreHandler = {
})
return callback(err)
}
callback(err, result.url, result.fileRef, true)
callback(err, result.url, result.fileRef, createdBlob)
}
)
}
@@ -159,7 +172,7 @@ const FileStoreHandler = {
const historyId = project.overleaf?.history?.id
const fileId = file._id
const hash = file.hash
if (historyId && hash) {
if (historyId && hash && Features.hasFeature('project-history-blobs')) {
// new behaviour - request from history
const range = _extractRange(query?.range)
HistoryManager.requestBlobWithFallback(
services/web/app/src/Features/History/HistoryController.js
+3
View File
@@ -77,6 +77,9 @@ module.exports = HistoryController = {
* has a hash.
* */
fileToBlobRedirectMiddleware(req, res, next) {
if (!Features.hasFeature('project-history-blobs')) {
return next()
}
const projectId = req.params.Project_id
const fileId = req.params.File_id
ProjectLocator.findElement(
services/web/app/src/Features/History/HistoryURLHelper.js
+2 -1
View File
@@ -7,7 +7,8 @@ function projectHistoryURLWithFilestoreFallback(
origin
) {
const filestoreURL = `${Settings.apis.filestore.url}/project/${projectId}/file/${fileRef._id}?from=${origin}`
if (fileRef.hash) {
// TODO: When this file is converted to ES modules we will be able to use Features.hasFeature('project-history-blobs'). Currently we can't stub the feature return value in tests.
if (fileRef.hash && Settings.enableProjectHistoryBlobs) {
return {
url: `${Settings.apis.project_history.url}/project/${historyId}/blob/${fileRef.hash}`,
fallbackURL: filestoreURL,
services/web/app/src/Features/Project/ProjectController.js
+3
View File
@@ -800,6 +800,9 @@ const _ProjectController = {
isInvitedMember
),
chatEnabled: Features.hasFeature('chat'),
projectHistoryBlobsEnabled: Features.hasFeature(
'project-history-blobs'
),
roMirrorOnClientNoLocalStorage:
Settings.adminOnlyLogin || project.name.startsWith('Debug: '),
languages: Settings.languages,
services/web/app/src/Features/Project/ProjectDuplicator.js
+21 -8
View File
@@ -213,7 +213,8 @@ async function _copyFiles(sourceEntries, sourceProject, targetProject) {
file.hash = sourceFile.hash
}
let createdBlob = false
if (file.hash != null) {
if (file.hash != null && Features.hasFeature('project-history-blobs')) {
const usingFilestore = Features.hasFeature('filestore')
try {
await HistoryManager.promises.copyBlob(
sourceHistoryId,
@@ -221,20 +222,32 @@ async function _copyFiles(sourceEntries, sourceProject, targetProject) {
file.hash
)
createdBlob = true
if (!usingFilestore) {
return { createdBlob, file, path, url: null }
}
} catch (err) {
logger.error(
{
err,
if (!usingFilestore) {
throw OError.tag(err, 'unexpected error copying blob', {
sourceProjectId: sourceProject._id,
targetProjectId: targetProject._id,
sourceFile,
sourceHistoryId,
},
'unexpected error copying blob'
)
})
} else {
logger.error(
{
err,
sourceProjectId: sourceProject._id,
targetProjectId: targetProject._id,
sourceFile,
sourceHistoryId,
},
'unexpected error copying blob'
)
}
}
}
if (createdBlob && Features.hasFeature('saas')) {
if (createdBlob && Features.hasFeature('project-history-blobs')) {
return { createdBlob, file, path, url: null }
}
const url = await FileStoreHandler.promises.copyFile(
services/web/app/src/Features/Project/ProjectEditorHandler.js
+8 -1
View File
@@ -1,6 +1,7 @@
let ProjectEditorHandler
const _ = require('lodash')
const Path = require('path')
const Features = require('../../infrastructure/Features')
function mergeDeletedDocs(a, b) {
const docIdsInA = new Set(a.map(doc => doc._id.toString()))
@@ -123,12 +124,18 @@ module.exports = ProjectEditorHandler = {
},
buildFileModelView(file) {
const additionalFileProperties = {}
if (Features.hasFeature('project-history-blobs')) {
additionalFileProperties.hash = file.hash
}
return {
_id: file._id,
name: file.name,
linkedFileData: file.linkedFileData,
created: file.created,
hash: file.hash,
...additionalFileProperties,
}
},
services/web/app/src/infrastructure/Features.js
+6
View File
@@ -19,6 +19,8 @@ const trackChangesModuleAvailable =
* @property {boolean | undefined} enableGithubSync
* @property {boolean | undefined} enableGitBridge
* @property {boolean | undefined} enableHomepage
* @property {boolean | undefined} enableProjectHistoryBlobs
* @property {boolean | undefined} disableFilestore
* @property {boolean | undefined} enableSaml
* @property {boolean | undefined} ldap
* @property {boolean | undefined} oauth
@@ -86,6 +88,10 @@ const Features = {
_.get(Settings, ['apis', 'linkedUrlProxy', 'url']) &&
Settings.enabledLinkedFileTypes.includes('url')
)
case 'project-history-blobs':
return Boolean(Settings.enableProjectHistoryBlobs)
case 'filestore':
return Boolean(Settings.disableFilestore) === false
case 'support':
return supportModuleAvailable
case 'symbol-palette':
services/web/app/views/project/editor/_meta.pug
+1
View File
@@ -10,6 +10,7 @@ meta(name="ol-isRestrictedTokenMember" data-type="boolean" content=isRestrictedT
meta(name="ol-maxDocLength" data-type="json" content=maxDocLength)
meta(name="ol-wikiEnabled" data-type="boolean" content=settings.proxyLearn)
meta(name="ol-chatEnabled" data-type="boolean" content=chatEnabled)
meta(name="ol-projectHistoryBlobsEnabled" data-type="boolean" content=projectHistoryBlobsEnabled)
meta(name="ol-gitBridgePublicBaseUrl" content=gitBridgePublicBaseUrl)
meta(name="ol-gitBridgeEnabled" data-type="boolean" content=gitBridgeEnabled)
meta(name="ol-compilesUserContentDomain" content=settings.compilesUserContentDomain)