agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-06-11 07:00:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

[filestore] remove user files endpoints (#28125)

Browse Source 
* [filestore] remove user files endpoints

* [web] remove user files integration for filestore

GitOrigin-RevId: 565fa68a659c07420ee6141d0f276b4e4d2972e0
This commit is contained in:
Jakob Ackermann
2025-09-01 11:43:37 +02:00
committed by Copybot
parent 21d3879574
commit 319a542e8d
73 changed files with 1481 additions and 3587 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/web/app/src/Features/Compile/ClsiManager.js
+1 -11
View File
@@ -15,7 +15,6 @@ const { Cookie } = require('tough-cookie')
const ClsiCookieManager = require('./ClsiCookieManager')(
Settings.apis.clsi?.backendGroupName
)
const Features = require('../../infrastructure/Features')
const NewBackendCloudClsiCookieManager = require('./ClsiCookieManager')(
Settings.apis.clsi_new?.backendGroupName
)
@@ -750,18 +749,9 @@ function _finaliseRequest(projectId, options, project, docs, files) {
for (let path in files) {
const file = files[path]
path = path.replace(/^\//, '') // Remove leading /
const filestoreURL = `${Settings.apis.filestore.url}/project/${project._id}/file/${file._id}`
let url = filestoreURL
let fallbackURL
if (file.hash && Features.hasFeature('project-history-blobs')) {
url = getFilestoreBlobURL(historyId, file.hash)
fallbackURL = filestoreURL
}
resources.push({
path,
url,
fallbackURL,
url: getFilestoreBlobURL(historyId, file.hash),
modified: file.created?.getTime(),
})
}
services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js
+15 -29
View File
@@ -8,8 +8,6 @@ const metrics = require('@overleaf/metrics')
const { promisify, callbackify } = require('util')
const { promisifyMultiResult } = require('@overleaf/promise-utils')
const ProjectGetter = require('../Project/ProjectGetter')
const FileStoreHandler = require('../FileStore/FileStoreHandler')
const Features = require('../../infrastructure/Features')
const Modules = require('../../infrastructure/Modules')
function getProjectLastUpdatedAt(projectId, callback) {
@@ -361,25 +359,19 @@ function resyncProjectHistory(
doc: doc.doc._id,
path: doc.path,
}))
const hasFilestore = Features.hasFeature('filestore')
if (!hasFilestore) {
// Files without a hash likely do not have a blob. Abort.
for (const { file } of files) {
if (!file.hash) {
return callback(
new OError('found file with missing hash', { projectId, file })
)
}
// Files without a hash likely do not have a blob. Abort.
for (const { file } of files) {
if (!file.hash) {
return callback(
new OError('found file with missing hash', { projectId, file })
)
}
}
files = files.map(file => ({
file: file.file._id,
path: file.path,
url: hasFilestore
? FileStoreHandler._buildUrl(projectId, file.file._id)
: undefined,
_hash: file.file.hash,
createdBlob: !hasFilestore,
createdBlob: true,
metadata: buildFileMetadataForHistory(file.file),
}))
@@ -480,15 +472,12 @@ function updateProjectStructure(
changes.newDocs,
historyRangesSupport
)
const hasFilestore = Features.hasFeature('filestore')
if (!hasFilestore) {
for (const newEntity of changes.newFiles || []) {
if (!newEntity.file.hash) {
// Files without a hash likely do not have a blob. Abort.
return callback(
new OError('found file with missing hash', { newEntity })
)
}
for (const newEntity of changes.newFiles || []) {
if (!newEntity.file.hash) {
// Files without a hash likely do not have a blob. Abort.
return callback(
new OError('found file with missing hash', { newEntity })
)
}
}
const {
@@ -623,8 +612,6 @@ function _getUpdates(
})
}
}
const hasFilestore = Features.hasFeature('filestore')
for (const id in newEntitiesHash) {
const newEntity = newEntitiesHash[id]
const oldEntity = oldEntitiesHash[id]
@@ -638,10 +625,9 @@ function _getUpdates(
docLines: newEntity.docLines,
ranges: newEntity.ranges,
historyRangesSupport,
url: newEntity.file != null && hasFilestore ? newEntity.url : undefined,
hash: newEntity.file != null ? newEntity.file.hash : undefined,
hash: newEntity.file?.hash,
metadata: buildFileMetadataForHistory(newEntity.file),
createdBlob: (newEntity.createdBlob || !hasFilestore) ?? false,
createdBlob: true,
})
} else if (newEntity.path !== oldEntity.path) {
// entity renamed
services/web/app/src/Features/Downloads/ProjectZipStreamManager.mjs
+10 -17
View File
@@ -4,8 +4,6 @@ import logger from '@overleaf/logger'
import ProjectEntityHandler from '../Project/ProjectEntityHandler.js'
import ProjectGetter from '../Project/ProjectGetter.js'
import HistoryManager from '../History/HistoryManager.js'
import FileStoreHandler from '../FileStore/FileStoreHandler.js'
import Features from '../../infrastructure/Features.js'
let ProjectZipStreamManager
export default ProjectZipStreamManager = {
@@ -111,22 +109,17 @@ export default ProjectZipStreamManager = {
},
getFileStream: (projectId, file, callback) => {
if (Features.hasFeature('project-history-blobs')) {
HistoryManager.requestBlobWithFallback(
projectId,
file.hash,
file._id,
(error, result) => {
if (error) {
return callback(error)
}
const { stream } = result
callback(null, stream)
HistoryManager.requestBlobWithProjectId(
projectId,
file.hash,
(error, result) => {
if (error) {
return callback(error)
}
)
} else {
FileStoreHandler.getFileStream(projectId, file._id, {}, callback)
}
const { stream } = result
callback(null, stream)
}
)
},
addAllFilesToArchive(projectId, archive, callback) {
services/web/app/src/Features/FileStore/FileStoreController.mjs
+14 -34
View File
@@ -4,11 +4,9 @@ import { pipeline } from 'node:stream/promises'
import logger from '@overleaf/logger'
import { expressify } from '@overleaf/promise-utils'
import Metrics from '@overleaf/metrics'
import FileStoreHandler from './FileStoreHandler.js'
import ProjectLocator from '../Project/ProjectLocator.js'
import HistoryManager from '../History/HistoryManager.js'
import Errors from '../Errors/Errors.js'
import Features from '../../infrastructure/Features.js'
import { preparePlainTextResponse } from '../../infrastructure/Response.js'
async function getFile(req, res) {
@@ -55,25 +53,15 @@ async function getFile(req, res) {
status: Boolean(file?.hash),
})
let source, stream, contentLength
let stream, contentLength
try {
if (Features.hasFeature('project-history-blobs') && file?.hash) {
// Get the file from history
;({ source, stream, contentLength } =
await HistoryManager.promises.requestBlobWithFallback(
projectId,
file.hash,
fileId
))
} else {
// The file-hash is missing. Fall back to filestore.
stream = await FileStoreHandler.promises.getFileStream(
// Get the file from history
;({ stream, contentLength } =
await HistoryManager.promises.requestBlobWithProjectId(
projectId,
fileId,
queryString
)
source = 'filestore'
}
file.hash,
'GET'
))
} catch (err) {
if (err instanceof Errors.NotFoundError) {
return res.status(404).end()
@@ -97,7 +85,6 @@ async function getFile(req, res) {
// allow the browser to cache these immutable files
// note: both "private" and "max-age" appear to be required for caching
res.setHeader('Cache-Control', 'private, max-age=3600')
res.appendHeader('X-Served-By', source)
try {
await pipeline(stream, res)
} catch (err) {
@@ -150,20 +137,14 @@ async function getFileHead(req, res) {
status: Boolean(file?.hash),
})
let fileSize, source
let fileSize
try {
if (Features.hasFeature('project-history-blobs') && file?.hash) {
;({ source, contentLength: fileSize } =
await HistoryManager.promises.requestBlobWithFallback(
projectId,
file.hash,
fileId,
'HEAD'
))
} else {
fileSize = await FileStoreHandler.promises.getFileSize(projectId, fileId)
source = 'filestore'
}
;({ contentLength: fileSize } =
await HistoryManager.promises.requestBlobWithProjectId(
projectId,
file.hash,
'HEAD'
))
} catch (err) {
if (err instanceof Errors.NotFoundError) {
return res.status(404).end()
@@ -174,7 +155,6 @@ async function getFileHead(req, res) {
}
res.setHeader('Content-Length', fileSize)
res.appendHeader('X-Served-By', source)
res.status(200).end()
}
services/web/app/src/Features/FileStore/FileStoreHandler.js
+10 -282
View File
@@ -1,20 +1,12 @@
const _ = require('lodash')
const logger = require('@overleaf/logger')
const fs = require('fs')
const request = require('request')
const settings = require('@overleaf/settings')
const Async = require('async')
const FileHashManager = require('./FileHashManager')
const HistoryManager = require('../History/HistoryManager')
const ProjectDetailsHandler = require('../Project/ProjectDetailsHandler')
const { File } = require('../../models/File')
const Errors = require('../Errors/Errors')
const OError = require('@overleaf/o-error')
const { promisifyAll } = require('@overleaf/promise-utils')
const Features = require('../../infrastructure/Features')
const ONE_MIN_IN_MS = 60 * 1000
const FIVE_MINS_IN_MS = ONE_MIN_IN_MS * 5
const FileStoreHandler = {
RETRY_ATTEMPTS: 3,
@@ -40,27 +32,14 @@ const FileStoreHandler = {
},
_uploadToHistory(historyId, hash, size, fsPath, callback) {
if (Features.hasFeature('project-history-blobs')) {
Async.retry(
FileStoreHandler.RETRY_ATTEMPTS,
cb =>
HistoryManager.uploadBlobFromDisk(historyId, hash, size, fsPath, cb),
error => {
if (error) return callback(error, false)
callback(null, true)
}
)
} else {
callback(null, false)
}
},
_uploadToFileStore(projectId, fileArgs, fsPath, callback) {
Async.retry(
FileStoreHandler.RETRY_ATTEMPTS,
(cb, results) =>
FileStoreHandler._doUploadFileFromDisk(projectId, fileArgs, fsPath, cb),
callback
cb =>
HistoryManager.uploadBlobFromDisk(historyId, hash, size, fsPath, cb),
error => {
if (error) return callback(error, false)
callback(null, true)
}
)
},
@@ -99,274 +78,23 @@ const FileStoreHandler = {
hash,
stat.size,
fsPath,
function (err, createdBlob) {
function (err) {
if (err) {
return callback(err)
}
fileArgs = { ...fileArgs, hash }
FileStoreHandler._uploadToFileStore(
projectId,
fileArgs,
fsPath,
function (err, result) {
if (err) {
OError.tag(err, 'Error uploading file, retries failed', {
projectId,
fileArgs,
})
return callback(err)
}
callback(err, result.url, result.fileRef, createdBlob)
}
)
callback(err, new File(fileArgs), true)
}
)
})
})
},
_doUploadFileFromDisk(projectId, fileArgs, fsPath, callback) {
const callbackOnce = _.once(callback)
const fileRef = new File(fileArgs)
const fileId = fileRef._id
const url = FileStoreHandler._buildUrl(projectId, fileId)
if (!Features.hasFeature('filestore')) {
return callbackOnce(null, { url, fileRef })
}
const readStream = fs.createReadStream(fsPath)
readStream.on('error', function (err) {
logger.warn(
{ err, projectId, fileId, fsPath },
'something went wrong on the read stream of uploadFileFromDisk'
)
callbackOnce(err)
})
readStream.on('open', function () {
const opts = {
method: 'post',
uri: url,
timeout: FIVE_MINS_IN_MS,
headers: {
'X-File-Hash-From-Web': fileArgs.hash,
}, // send the hash to the filestore as a custom header so it can be checked
}
const writeStream = request(opts)
writeStream.on('error', function (err) {
logger.warn(
{ err, projectId, fileId, fsPath },
'something went wrong on the write stream of uploadFileFromDisk'
)
callbackOnce(err)
})
writeStream.on('response', function (response) {
if (![200, 201].includes(response.statusCode)) {
const err = new OError(
`non-ok response from filestore for upload: ${response.statusCode}`,
{ statusCode: response.statusCode }
)
return callbackOnce(err)
}
callbackOnce(null, { url, fileRef })
}) // have to pass back an object because async.retry only accepts a single result argument
readStream.pipe(writeStream)
})
},
getFileStreamNew(project, file, query, callback) {
const projectId = project._id
const historyId = project.overleaf?.history?.id
const fileId = file._id
const hash = file.hash
if (historyId && hash && Features.hasFeature('project-history-blobs')) {
// new behaviour - request from history
const range = _extractRange(query?.range)
HistoryManager.requestBlobWithFallback(
projectId,
hash,
fileId,
'GET',
range,
function (err, result) {
if (err) {
return callback(err)
}
const { stream } = result
callback(null, stream)
}
)
} else {
// original behaviour
FileStoreHandler.getFileStream(projectId, fileId, query, callback)
}
},
getFileStream(projectId, fileId, query, callback) {
if (!Features.hasFeature('filestore')) {
return callback(
new Errors.NotFoundError('filestore is disabled, file not found')
)
}
let queryString = '?from=getFileStream'
if (query != null && query.format != null) {
queryString += `&format=${query.format}`
}
const opts = {
method: 'get',
uri: `${this._buildUrl(projectId, fileId)}${queryString}`,
timeout: FIVE_MINS_IN_MS,
headers: {},
}
if (query != null && query.range != null) {
const rangeText = query.range
if (rangeText && rangeText.match != null && rangeText.match(/\d+-\d+/)) {
opts.headers.range = `bytes=${query.range}`
}
}
const readStream = request(opts)
readStream.on('error', err =>
logger.err(
{ err, projectId, fileId, query, opts },
'error in file stream'
)
)
callback(null, readStream)
},
getFileSize(projectId, fileId, callback) {
const url = this._buildUrl(projectId, fileId)
request.head(`${url}?from=getFileSize`, (err, res) => {
if (err) {
OError.tag(err, 'failed to get file size from filestore', {
projectId,
fileId,
})
return callback(err)
}
if (res.statusCode === 404) {
return callback(new Errors.NotFoundError('file not found in filestore'))
}
if (res.statusCode !== 200) {
logger.warn(
{ projectId, fileId, statusCode: res.statusCode },
'filestore returned non-200 response'
)
return callback(new Error('filestore returned non-200 response'))
}
const fileSize = res.headers['content-length']
callback(null, fileSize)
})
},
deleteFile(projectId, fileId, callback) {
logger.debug({ projectId, fileId }, 'telling file store to delete file')
const opts = {
method: 'delete',
uri: this._buildUrl(projectId, fileId),
timeout: FIVE_MINS_IN_MS,
}
request(opts, function (err, response) {
if (err) {
logger.warn(
{ err, projectId, fileId },
'something went wrong deleting file from filestore'
)
}
callback(err)
})
},
deleteProject(projectId, callback) {
if (!Features.hasFeature('filestore')) {
return callback() // if filestore is not in use, we don't need to delete anything
}
request(
{
method: 'delete',
uri: this._buildUrl(projectId),
timeout: FIVE_MINS_IN_MS,
},
err => {
if (err) {
return callback(
OError.tag(
err,
'something went wrong deleting a project in filestore',
{ projectId }
)
)
}
callback()
}
)
},
copyFile(oldProjectId, oldFileId, newProjectId, newFileId, callback) {
logger.debug(
{ oldProjectId, oldFileId, newProjectId, newFileId },
'telling filestore to copy a file'
)
const opts = {
method: 'put',
json: {
source: {
project_id: oldProjectId,
file_id: oldFileId,
},
},
uri: this._buildUrl(newProjectId, newFileId),
timeout: FIVE_MINS_IN_MS,
}
request(opts, function (err, response) {
if (err) {
OError.tag(
err,
'something went wrong telling filestore api to copy file',
{
oldProjectId,
oldFileId,
newProjectId,
newFileId,
}
)
callback(err)
} else if (response.statusCode >= 200 && response.statusCode < 300) {
// successful response
callback(null, opts.uri)
} else {
err = new OError(
`non-ok response from filestore for copyFile: ${response.statusCode}`,
{
uri: opts.uri,
statusCode: response.statusCode,
}
)
callback(err)
}
})
},
_buildUrl(projectId, fileId) {
return (
`${settings.apis.filestore.url}/project/${projectId}` +
(fileId ? `/file/${fileId}` : '')
)
},
}
function _extractRange(range) {
if (typeof range === 'string' && /\d+-\d+/.test(range)) {
return `bytes=${range}`
}
}
module.exports = FileStoreHandler
module.exports.promises = promisifyAll(FileStoreHandler, {
multiResult: {
uploadFileFromDisk: ['url', 'fileRef', 'createdBlob'],
uploadFileFromDiskWithHistoryId: ['url', 'fileRef', 'createdBlob'],
uploadFileFromDisk: ['fileRef', 'createdBlob'],
uploadFileFromDiskWithHistoryId: ['fileRef', 'createdBlob'],
},
})
services/web/app/src/Features/History/HistoryController.js
+3 -5
View File
@@ -52,13 +52,12 @@ async function requestBlob(method, req, res) {
}
const range = req.get('Range')
let stream, source, contentLength
let stream, contentLength
try {
;({ stream, source, contentLength } =
await HistoryManager.promises.requestBlobWithFallback(
;({ stream, contentLength } =
await HistoryManager.promises.requestBlobWithProjectId(
projectId,
hash,
req.query.fallback,
method,
range
))
@@ -66,7 +65,6 @@ async function requestBlob(method, req, res) {
if (err instanceof Errors.NotFoundError) return res.status(404).end()
throw err
}
res.appendHeader('X-Served-By', source)
if (contentLength) res.setHeader('Content-Length', contentLength) // set on HEAD
res.setHeader('Content-Type', 'application/octet-stream')
services/web/app/src/Features/History/HistoryManager.js
+16 -32
View File
@@ -11,9 +11,8 @@ const OError = require('@overleaf/o-error')
const UserGetter = require('../User/UserGetter')
const ProjectGetter = require('../Project/ProjectGetter')
const HistoryBackupDeletionHandler = require('./HistoryBackupDeletionHandler')
const { db, ObjectId, waitForDb } = require('../../infrastructure/mongodb')
const { db, waitForDb } = require('../../infrastructure/mongodb')
const Metrics = require('@overleaf/metrics')
const logger = require('@overleaf/logger')
const { NotFoundError } = require('../Errors/Errors')
const HISTORY_V1_URL = settings.apis.v1_history.url
@@ -169,22 +168,25 @@ async function copyBlob(sourceHistoryId, targetHistoryId, hash) {
)
}
async function requestBlobWithFallback(
async function requestBlobWithProjectId(
projectId,
hash,
fileId,
method = 'GET',
range = ''
) {
const project = await ProjectGetter.promises.getProject(projectId, {
'overleaf.history.id': true,
})
return requestBlob(project.overleaf.history.id, hash, method, range)
}
async function requestBlob(historyId, hash, method = 'GET', range = '') {
// Talk to history-v1 directly to avoid streaming via project-history.
let url = new URL(HISTORY_V1_URL)
url.pathname += `/projects/${project.overleaf.history.id}/blobs/${hash}`
const url = new URL(HISTORY_V1_URL)
url.pathname += `/projects/${historyId}/blobs/${hash}`
const opts = { method, headers: { Range: range } }
let stream, response, source
let stream, response
try {
;({ stream, response } = await fetchStreamWithResponse(url, {
...opts,
@@ -193,38 +195,18 @@ async function requestBlobWithFallback(
password: settings.apis.v1_history.pass,
},
}))
source = 'history-v1'
} catch (err) {
if (err instanceof RequestFailedError && err.response.status === 404) {
if (ObjectId.isValid(fileId)) {
url = new URL(settings.apis.filestore.url)
url.pathname = `/project/${projectId}/file/${fileId}`
try {
;({ stream, response } = await fetchStreamWithResponse(url, opts))
} catch (err) {
if (
err instanceof RequestFailedError &&
err.response.status === 404
) {
throw new NotFoundError()
}
throw err
}
logger.warn({ projectId, hash, fileId }, 'missing history blob')
source = 'filestore'
} else {
throw new NotFoundError()
}
throw new NotFoundError()
} else {
throw err
}
}
Metrics.inc('request_blob', 1, { path: source })
Metrics.inc('request_blob', 1, { path: 'history-v1' })
return {
url,
stream,
source,
contentLength: response.headers.get('Content-Length'),
contentLength: parseInt(response.headers.get('Content-Length'), 10),
}
}
@@ -417,7 +399,8 @@ module.exports = {
getCurrentContent: callbackify(getCurrentContent),
uploadBlobFromDisk: callbackify(uploadBlobFromDisk),
copyBlob: callbackify(copyBlob),
requestBlobWithFallback: callbackify(requestBlobWithFallback),
requestBlob: callbackify(requestBlob),
requestBlobWithProjectId: callbackify(requestBlobWithProjectId),
getLatestHistory: callbackify(getLatestHistory),
getChanges: callbackify(getChanges),
promises: {
@@ -431,7 +414,8 @@ module.exports = {
getContentAtVersion,
uploadBlobFromDisk,
copyBlob,
requestBlobWithFallback,
requestBlob,
requestBlobWithProjectId,
getLatestHistory,
getChanges,
},
services/web/app/src/Features/History/HistoryURLHelper.js
-21
View File
@@ -1,21 +0,0 @@
// Pass settings to enable consistent unit tests from .js and .mjs modules
function projectHistoryURLWithFilestoreFallback(
Settings,
projectId,
historyId,
fileRef,
origin
) {
const filestoreURL = `${Settings.apis.filestore.url}/project/${projectId}/file/${fileRef._id}?from=${origin}`
// TODO: When this file is converted to ES modules we will be able to use Features.hasFeature('project-history-blobs'). Currently we can't stub the feature return value in tests.
if (fileRef.hash && Settings.filestoreMigrationLevel >= 1) {
return {
url: `${Settings.apis.project_history.url}/project/${historyId}/blob/${fileRef.hash}`,
fallbackURL: filestoreURL,
}
} else {
return { url: filestoreURL }
}
}
module.exports = { projectHistoryURLWithFilestoreFallback }
services/web/app/src/Features/LinkedFiles/ProjectFileAgent.js
+7 -7
View File
@@ -15,7 +15,6 @@ const AuthorizationManager = require('../Authorization/AuthorizationManager')
const ProjectLocator = require('../Project/ProjectLocator')
const DocstoreManager = require('../Docstore/DocstoreManager')
const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
const FileStoreHandler = require('../FileStore/FileStoreHandler')
const _ = require('lodash')
const LinkedFilesHandler = require('./LinkedFilesHandler')
const {
@@ -25,6 +24,7 @@ const {
SourceFileNotFoundError,
} = require('./LinkedFilesErrors')
const { promisify } = require('@overleaf/promise-utils')
const HistoryManager = require('../History/HistoryManager')
module.exports = ProjectFileAgent = {
createLinkedFile(
@@ -134,17 +134,17 @@ module.exports = ProjectFileAgent = {
}
) // Created
} else if (type === 'file') {
return FileStoreHandler.getFileStreamNew(
sourceProject,
entity,
null,
function (err, fileStream) {
return HistoryManager.requestBlob(
sourceProject.overleaf.history.id,
entity.hash,
'GET',
function (err, result) {
if (err != null) {
return callback(err)
}
return LinkedFilesHandler.importFromStream(
projectId,
fileStream,
result.stream,
linkedFileData,
name,
parentFolderId,
services/web/app/src/Features/Project/ProjectController.js
-3
View File
@@ -844,9 +844,6 @@ const _ProjectController = {
isInvitedMember
),
capabilities,
projectHistoryBlobsEnabled: Features.hasFeature(
'project-history-blobs'
),
roMirrorOnClientNoLocalStorage:
Settings.adminOnlyLogin || project.name.startsWith('Debug: '),
languages: Settings.languages,
services/web/app/src/Features/Project/ProjectDeleter.js
-2
View File
@@ -16,7 +16,6 @@ const CollaboratorsGetter = require('../Collaborators/CollaboratorsGetter')
const DocstoreManager = require('../Docstore/DocstoreManager')
const EditorRealTimeController = require('../Editor/EditorRealTimeController')
const HistoryManager = require('../History/HistoryManager')
const FilestoreHandler = require('../FileStore/FileStoreHandler')
const ChatApiHandler = require('../Chat/ChatApiHandler')
const { promiseMapWithLimit } = require('@overleaf/promise-utils')
const { READ_PREFERENCE_SECONDARY } = require('../../infrastructure/mongodb')
@@ -350,7 +349,6 @@ async function expireDeletedProject(projectId) {
deletedProject.project._id,
historyId
),
FilestoreHandler.promises.deleteProject(deletedProject.project._id),
ChatApiHandler.promises.destroyProject(deletedProject.project._id),
ProjectAuditLogEntry.deleteMany({ projectId }),
Modules.promises.hooks.fire('projectExpired', deletedProject.project._id),
services/web/app/src/Features/Project/ProjectDuplicator.js
+17 -56
View File
@@ -7,7 +7,6 @@ const { Doc } = require('../../models/Doc')
const { File } = require('../../models/File')
const DocstoreManager = require('../Docstore/DocstoreManager')
const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
const FileStoreHandler = require('../FileStore/FileStoreHandler')
const HistoryManager = require('../History/HistoryManager')
const ProjectCreationHandler = require('./ProjectCreationHandler')
const ProjectDeleter = require('./ProjectDeleter')
@@ -20,7 +19,6 @@ const SafePath = require('./SafePath')
const TpdsProjectFlusher = require('../ThirdPartyDataStore/TpdsProjectFlusher')
const _ = require('lodash')
const TagsHandler = require('../Tags/TagsHandler')
const Features = require('../../infrastructure/Features')
const ClsiCacheManager = require('../Compile/ClsiCacheManager')
module.exports = {
@@ -225,66 +223,29 @@ async function _copyFiles(sourceEntries, sourceProject, targetProject) {
async sourceEntry => {
const sourceFile = sourceEntry.file
const path = sourceEntry.path
const file = new File({ name: SafePath.clean(sourceFile.name) })
const file = new File({
name: SafePath.clean(sourceFile.name),
hash: sourceFile.hash,
})
if (sourceFile.linkedFileData != null) {
file.linkedFileData = sourceFile.linkedFileData
file.created = sourceFile.created
}
if (sourceFile.hash != null) {
file.hash = sourceFile.hash
}
let createdBlob = false
const usingFilestore = Features.hasFeature('filestore')
if (file.hash != null && Features.hasFeature('project-history-blobs')) {
try {
await HistoryManager.promises.copyBlob(
sourceHistoryId,
targetHistoryId,
file.hash
)
createdBlob = true
if (!usingFilestore) {
return { createdBlob, file, path, url: null }
}
} catch (err) {
if (!usingFilestore) {
throw OError.tag(err, 'unexpected error copying blob', {
sourceProjectId: sourceProject._id,
targetProjectId: targetProject._id,
sourceFile,
sourceHistoryId,
})
} else {
logger.error(
{
err,
sourceProjectId: sourceProject._id,
targetProjectId: targetProject._id,
sourceFile,
sourceHistoryId,
},
'unexpected error copying blob'
)
}
}
}
if (createdBlob && Features.hasFeature('project-history-blobs')) {
return { createdBlob, file, path, url: null }
}
if (!usingFilestore) {
// Note: This is also checked in app.mjs
throw new OError(
'bad config: need to enable either filestore or project-history-blobs'
try {
await HistoryManager.promises.copyBlob(
sourceHistoryId,
targetHistoryId,
file.hash
)
return { createdBlob: true, file, path }
} catch (err) {
throw OError.tag(err, 'unexpected error copying blob', {
sourceProjectId: sourceProject._id,
targetProjectId: targetProject._id,
sourceFile,
sourceHistoryId,
})
}
const url = await FileStoreHandler.promises.copyFile(
sourceProject._id,
sourceFile._id,
targetProject._id,
file._id
)
return { createdBlob, file, path, url }
}
)
return targetEntries
services/web/app/src/Features/Project/ProjectEditorHandler.js
+1 -8
View File
@@ -1,7 +1,6 @@
let ProjectEditorHandler
const _ = require('lodash')
const Path = require('path')
const Features = require('../../infrastructure/Features')
module.exports = ProjectEditorHandler = {
trackChangesAvailable: false,
@@ -98,18 +97,12 @@ module.exports = ProjectEditorHandler = {
},
buildFileModelView(file) {
const additionalFileProperties = {}
if (Features.hasFeature('project-history-blobs')) {
additionalFileProperties.hash = file.hash
}
return {
_id: file._id,
name: file.name,
linkedFileData: file.linkedFileData,
created: file.created,
...additionalFileProperties,
hash: file.hash,
}
},
services/web/app/src/Features/Project/ProjectEntityUpdateHandler.js
+15 -36
View File
@@ -333,7 +333,7 @@ const addFile = wrapWithLock({
if (!SafePath.isCleanFilename(fileName)) {
throw new Errors.InvalidNameError('invalid element name')
}
const { url, fileRef, createdBlob } =
const { fileRef, createdBlob } =
await ProjectEntityUpdateHandler._uploadFile(
projectId,
folderId,
@@ -347,7 +347,6 @@ const addFile = wrapWithLock({
folderId,
userId,
fileRef,
fileStoreUrl: url,
createdBlob,
source,
}
@@ -357,7 +356,6 @@ const addFile = wrapWithLock({
folderId,
userId,
fileRef,
fileStoreUrl,
createdBlob,
source,
}) {
@@ -374,7 +372,6 @@ const addFile = wrapWithLock({
createdBlob,
file: fileRef,
path: result && result.path && result.path.fileSystem,
url: fileStoreUrl,
},
]
await DocumentUpdaterHandler.promises.updateProjectStructure(
@@ -548,7 +545,7 @@ const upsertFile = wrapWithLock({
name: fileName,
linkedFileData,
}
const { url, fileRef, createdBlob } =
const { fileRef, createdBlob } =
await FileStoreHandler.promises.uploadFileFromDisk(
projectId,
fileArgs,
@@ -563,7 +560,6 @@ const upsertFile = wrapWithLock({
linkedFileData,
userId,
fileRef,
fileStoreUrl: url,
createdBlob,
source,
}
@@ -574,7 +570,6 @@ const upsertFile = wrapWithLock({
fileName,
userId,
fileRef,
fileStoreUrl,
createdBlob,
source,
}) {
@@ -639,7 +634,6 @@ const upsertFile = wrapWithLock({
createdBlob,
file: fileRef,
path: path.fileSystem,
url: fileStoreUrl,
},
],
newProject: project,
@@ -659,7 +653,6 @@ const upsertFile = wrapWithLock({
existingFile._id,
userId,
fileRef,
fileStoreUrl,
folderId,
source,
createdBlob
@@ -673,7 +666,6 @@ const upsertFile = wrapWithLock({
folderId,
userId,
fileRef,
fileStoreUrl,
createdBlob,
source,
})
@@ -733,15 +725,12 @@ const upsertFileWithPath = wrapWithLock({
name: fileName,
linkedFileData,
}
const {
url: fileStoreUrl,
fileRef,
createdBlob,
} = await FileStoreHandler.promises.uploadFileFromDisk(
projectId,
fileArgs,
fsPath
)
const { fileRef, createdBlob } =
await FileStoreHandler.promises.uploadFileFromDisk(
projectId,
fileArgs,
fsPath
)
return {
projectId,
@@ -751,7 +740,6 @@ const upsertFileWithPath = wrapWithLock({
linkedFileData,
userId,
fileRef,
fileStoreUrl,
createdBlob,
source,
}
@@ -764,7 +752,6 @@ const upsertFileWithPath = wrapWithLock({
linkedFileData,
userId,
fileRef,
fileStoreUrl,
createdBlob,
source,
}) {
@@ -787,7 +774,6 @@ const upsertFileWithPath = wrapWithLock({
linkedFileData,
userId,
fileRef,
fileStoreUrl,
createdBlob,
source,
})
@@ -1084,15 +1070,12 @@ const convertDocToFile = wrapWithLock({
}
await DocumentUpdaterHandler.promises.deleteDoc(projectId, docId, false)
const fsPath = await FileWriter.promises.writeLinesToDisk(projectId, lines)
const {
url: fileStoreUrl,
fileRef,
createdBlob,
} = await FileStoreHandler.promises.uploadFileFromDisk(
projectId,
{ name: doc.name, rev: rev + 1 },
fsPath
)
const { fileRef, createdBlob } =
await FileStoreHandler.promises.uploadFileFromDisk(
projectId,
{ name: doc.name, rev: rev + 1 },
fsPath
)
try {
await fs.promises.unlink(fsPath)
} catch (err) {
@@ -1103,7 +1086,6 @@ const convertDocToFile = wrapWithLock({
doc,
path: docPath,
fileRef,
fileStoreUrl,
userId,
source,
createdBlob,
@@ -1114,7 +1096,6 @@ const convertDocToFile = wrapWithLock({
doc,
path,
fileRef,
fileStoreUrl,
userId,
source,
createdBlob,
@@ -1133,7 +1114,7 @@ const convertDocToFile = wrapWithLock({
userId,
{
oldDocs: [{ doc, path }],
newFiles: [{ file: fileRef, path, url: fileStoreUrl, createdBlob }],
newFiles: [{ file: fileRef, path, createdBlob }],
newProject: project,
},
source
@@ -1380,7 +1361,6 @@ const ProjectEntityUpdateHandler = {
fileId,
userId,
newFileRef,
fileStoreUrl,
folderId,
source,
createdBlob
@@ -1409,7 +1389,6 @@ const ProjectEntityUpdateHandler = {
file: updatedFileRef,
createdBlob,
path: path.fileSystem,
url: fileStoreUrl,
},
]
const projectHistoryId = project.overleaf?.history?.id
services/web/app/src/Features/References/ReferencesHandler.mjs
+4 -11
View File
@@ -24,7 +24,6 @@ import _ from 'lodash'
import Async from 'async'
import Errors from '../Errors/Errors.js'
import { promisify } from '@overleaf/promise-utils'
import HistoryURLHelper from '../History/HistoryURLHelper.js'
let ReferencesHandler
@@ -167,21 +166,15 @@ export default ReferencesHandler = {
const bibDocUrls = docIds.map(docId =>
ReferencesHandler._buildDocUrl(projectId, docId)
)
const bibFileUrls = fileRefs.map(fileRef =>
HistoryURLHelper.projectHistoryURLWithFilestoreFallback(
settings,
projectId,
historyId,
fileRef,
'bibFileUrls'
)
)
const bibFileUrls = fileRefs.map(fileRef => ({
url: `${settings.apis.project_history.url}/project/${historyId}/blob/${fileRef.hash}`,
}))
const sourceURLs = bibDocUrls.concat(bibFileUrls)
return request.post(
{
url: `${settings.apis.references.url}/project/${projectId}/index`,
json: {
docUrls: sourceURLs.map(item => item.fallbackURL || item.url),
docUrls: sourceURLs.map(item => item.url),
sourceURLs,
fullIndex: isFullIndex,
},
services/web/app/src/Features/ThirdPartyDataStore/TpdsUpdateSender.js
+2 -13
View File
@@ -6,7 +6,6 @@ const metrics = require('@overleaf/metrics')
const Path = require('path')
const { fetchNothing } = require('@overleaf/fetch-utils')
const settings = require('@overleaf/settings')
const HistoryURLHelper = require('../History/HistoryURLHelper')
const CollaboratorsGetter =
require('../Collaborators/CollaboratorsGetter').promises
@@ -81,24 +80,14 @@ async function addFile(params) {
rev,
folderId,
} = params
// Go through project-history to avoid the need for handling history-v1 authentication.
const { url, fallbackURL } =
HistoryURLHelper.projectHistoryURLWithFilestoreFallback(
settings,
projectId,
historyId,
{ _id: fileId, hash },
'tpdsAddFile'
)
await addEntity({
projectId,
path,
projectName,
rev,
folderId,
streamOrigin: url,
streamFallback: fallbackURL,
// Go through project-history to avoid the need for handling history-v1 authentication.
streamOrigin: `${settings.apis.project_history.url}/project/${historyId}/blob/${hash}`,
entityId: fileId,
entityType: 'file',
})
services/web/app/src/Features/Uploads/ProjectUploadManager.js
+2 -2
View File
@@ -194,14 +194,14 @@ async function _createFile(project, projectPath, fsPath) {
throw new OError('missing history id')
}
const fileName = Path.basename(projectPath)
const { createdBlob, fileRef, url } =
const { createdBlob, fileRef } =
await FileStoreHandler.promises.uploadFileFromDiskWithHistoryId(
projectId,
historyId,
{ name: fileName },
fsPath
)
return { createdBlob, file: fileRef, path: projectPath, url }
return { createdBlob, file: fileRef, path: projectPath }
}
async function _notifyDocumentUpdater(project, userId, changes) {
services/web/app/src/infrastructure/Features.js
-13
View File
@@ -19,7 +19,6 @@ const trackChangesModuleAvailable =
* @property {boolean | undefined} enableGithubSync
* @property {boolean | undefined} enableGitBridge
* @property {boolean | undefined} enableHomepage
* @property {number} filestoreMigrationLevel
* @property {boolean | undefined} enableSaml
* @property {boolean | undefined} ldap
* @property {boolean | undefined} oauth
@@ -29,14 +28,6 @@ const trackChangesModuleAvailable =
*/
const Features = {
validateSettings() {
if (![0, 1, 2].includes(Settings.filestoreMigrationLevel)) {
throw new Error(
`invalid OVERLEAF_FILESTORE_MIGRATION_LEVEL=${Settings.filestoreMigrationLevel}, expected 0, 1 or 2`
)
}
},
/**
* @returns {boolean}
*/
@@ -97,10 +88,6 @@ const Features = {
_.get(Settings, ['apis', 'linkedUrlProxy', 'url']) &&
Settings.enabledLinkedFileTypes.includes('url')
)
case 'project-history-blobs':
return Settings.filestoreMigrationLevel > 0
case 'filestore':
return Settings.filestoreMigrationLevel < 2
case 'support':
return supportModuleAvailable
case 'symbol-palette':