mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-12 13:13:45 +01:00
[web] read a linked project file's source from history (#36817)
* [web] read a linked project file's source from history Behind the `linked-file-from-history` flag, a linked `project_file` import reads the source out of the source project's latest history snapshot rather than out of its file tree in mongo, where docs and files are two kinds of thing. - `ProjectFileAgent` gains `_importFromHistory` alongside the existing `_importFromProject`, and the flag picks between them. The snapshot only holds what has reached history, so the source project is flushed through document-updater and project-history first. A file that is editable is loaded eagerly, which applies the edit operations sitting on top of its blob; one that is not keeps its hash and is streamed. The file is promisified on the way. - `History/HistoryOTContentManager.mjs` reads the latest chunk and rolls its changes into a snapshot, retrying the read. `History/HistoryBlobStore.mjs` is the blob store overleaf-editor-core asks for while loading a file. - `LinkedFilesController` resolves the flag, including from the referer: the import endpoints have no query string of their own. Every agent takes the assignment so that the ones that do not read it still pass it along. - `linked-file-from-history.spec.ts` drives the flag from the editor url, over three imports from another project: a file uploaded to the source project and then refreshed from it, an edit the source project has only made in its editor, and a binary file. Both sources produce the same file, so which one was read is asserted in the acceptance tests rather than here. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * [saas-e2e] add test coverage for tracked-delete in linked file source * Apply suggestions from code review Co-authored-by: Mathias Jakobsen <mathias.jakobsen@overleaf.com> --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Mathias Jakobsen <mathias.jakobsen@overleaf.com> GitOrigin-RevId: 3907d6d399e4b5315e68bf833663c06bc72f4011
This commit is contained in:
committed by
Copybot
co-authored by
Claude Opus 5
Mathias Jakobsen
parent
34d57a87f9
commit
f671c3dc59
@@ -0,0 +1,40 @@
|
||||
import { text } from 'node:stream/consumers'
|
||||
import HistoryManager from './HistoryManager.mjs'
|
||||
|
||||
/**
|
||||
* The blob store of a project's history, as overleaf-editor-core wants it.
|
||||
*
|
||||
* Reading covers what File#load asks for, so a snapshot's files can be loaded
|
||||
* eagerly here -- needed for the ones with pending edit operations, whose content
|
||||
* is not in a single blob.
|
||||
*/
|
||||
class HistoryBlobStore {
|
||||
/**
|
||||
* @param {string} historyId
|
||||
*/
|
||||
constructor(historyId) {
|
||||
this.historyId = historyId
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} hash
|
||||
* @return {Promise<string>}
|
||||
*/
|
||||
async getString(hash) {
|
||||
const { stream } = await HistoryManager.promises.requestBlob(
|
||||
this.historyId,
|
||||
hash
|
||||
)
|
||||
return await text(stream)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} hash
|
||||
* @return {Promise<object>}
|
||||
*/
|
||||
async getObject(hash) {
|
||||
return JSON.parse(await this.getString(hash))
|
||||
}
|
||||
}
|
||||
|
||||
export default HistoryBlobStore
|
||||
@@ -0,0 +1,68 @@
|
||||
import { setTimeout } from 'node:timers/promises'
|
||||
import { Chunk } from 'overleaf-editor-core'
|
||||
import logger from '@overleaf/logger'
|
||||
import Metrics from '@overleaf/metrics'
|
||||
import HistoryManager from './HistoryManager.mjs'
|
||||
|
||||
/**
|
||||
* @import { Snapshot } from 'overleaf-editor-core'
|
||||
*/
|
||||
|
||||
const READ_ATTEMPTS = 3
|
||||
const READ_RETRY_DELAY_MS = 100
|
||||
|
||||
/**
|
||||
* Read from history, repeating the request if it fails.
|
||||
*
|
||||
* Retried on any failure, an error status included: history answering 500 to one
|
||||
* request says no more about the next one than a connection that was dropped
|
||||
* does.
|
||||
*
|
||||
* @template T
|
||||
* @param {string} metricName
|
||||
* @param {string} msg - logged with the error on each retry
|
||||
* @param {object} info - logged with the error on each retry
|
||||
* @param {() => Promise<T>} fn
|
||||
* @return {Promise<T>}
|
||||
*/
|
||||
async function withRetries(metricName, msg, info, fn) {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
return await fn()
|
||||
} catch (err) {
|
||||
if (attempt === READ_ATTEMPTS) throw err
|
||||
Metrics.inc(metricName)
|
||||
logger.warn({ err, attempt, ...info }, `${msg}, retrying`)
|
||||
await setTimeout(attempt * READ_RETRY_DELAY_MS)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the project's latest chunk.
|
||||
*
|
||||
* @param {string} historyId
|
||||
* @return {Promise<Chunk>}
|
||||
*/
|
||||
async function fetchLatestChunk(historyId) {
|
||||
const { chunk } = await withRetries(
|
||||
'history_ot_chunk_fetch_retry',
|
||||
'failed to get the latest history chunk',
|
||||
{ historyId },
|
||||
() => HistoryManager.promises.getLatestHistoryWithHistoryId(historyId)
|
||||
)
|
||||
return Chunk.fromRaw(chunk)
|
||||
}
|
||||
|
||||
/**
|
||||
* The project's latest snapshot, and the version it is of.
|
||||
*
|
||||
* @param {string} historyId
|
||||
* @return {Promise<{snapshot: Snapshot, version: number}>}
|
||||
*/
|
||||
export async function getLatestSnapshot(historyId) {
|
||||
const chunk = await fetchLatestChunk(historyId)
|
||||
const snapshot = chunk.getSnapshot()
|
||||
snapshot.applyAll(chunk.getChanges())
|
||||
return { snapshot, version: chunk.getEndVersion() }
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
TooManyFilesError,
|
||||
} from '../Errors/Errors.js'
|
||||
import Modules from '../../infrastructure/Modules.mjs'
|
||||
import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs'
|
||||
import { plainTextResponse } from '../../infrastructure/Response.mjs'
|
||||
import { z, zz, parseReq } from '../../infrastructure/Validation.mjs'
|
||||
import EditorRealTimeController from '../Editor/EditorRealTimeController.mjs'
|
||||
@@ -177,13 +178,21 @@ async function createLinkedFile(req, res, next) {
|
||||
data.provider = provider
|
||||
data.importedAt = new Date().toISOString()
|
||||
|
||||
const historySource = await SplitTestHandler.promises.featureFlagEnabled(
|
||||
req,
|
||||
res,
|
||||
'linked-file-from-history',
|
||||
{ includeReferer: true }
|
||||
)
|
||||
|
||||
try {
|
||||
const newFileId = await Agent.promises.createLinkedFile(
|
||||
projectId,
|
||||
data,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
userId,
|
||||
historySource
|
||||
)
|
||||
if (name.endsWith('.bib')) {
|
||||
AnalyticsManager.recordEventForSession(req.session, 'linked-bib-file', {
|
||||
@@ -229,6 +238,14 @@ async function refreshLinkedFile(req, res, next) {
|
||||
}
|
||||
|
||||
linkedFileData.importedAt = new Date().toISOString()
|
||||
|
||||
const historySource = await SplitTestHandler.promises.featureFlagEnabled(
|
||||
req,
|
||||
res,
|
||||
'linked-file-from-history',
|
||||
{ includeReferer: true }
|
||||
)
|
||||
|
||||
let newFileId
|
||||
try {
|
||||
newFileId = await Agent.promises.refreshLinkedFile(
|
||||
@@ -236,7 +253,8 @@ async function refreshLinkedFile(req, res, next) {
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
userId,
|
||||
historySource
|
||||
)
|
||||
} catch (err) {
|
||||
return LinkedFilesController.handleError(err, req, res, next)
|
||||
|
||||
@@ -1,15 +1,3 @@
|
||||
/* eslint-disable
|
||||
n/handle-callback-err,
|
||||
max-len,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
import AuthorizationManager from '../Authorization/AuthorizationManager.mjs'
|
||||
import ProjectLocator from '../Project/ProjectLocator.mjs'
|
||||
import DocstoreManager from '../Docstore/DocstoreManager.mjs'
|
||||
@@ -17,8 +5,10 @@ import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mj
|
||||
import _ from 'lodash'
|
||||
import LinkedFilesHandler from './LinkedFilesHandler.mjs'
|
||||
import LinkedFilesErrors from './LinkedFilesErrors.mjs'
|
||||
import { promisify } from '@overleaf/promise-utils'
|
||||
import { callbackify } from '@overleaf/promise-utils'
|
||||
import HistoryManager from '../History/HistoryManager.mjs'
|
||||
import HistoryBlobStore from '../History/HistoryBlobStore.mjs'
|
||||
import * as HistoryOTContentManager from '../History/HistoryOTContentManager.mjs'
|
||||
import Errors from '../Errors/Errors.js'
|
||||
|
||||
const {
|
||||
@@ -28,191 +18,221 @@ const {
|
||||
SourceFileNotFoundError,
|
||||
} = LinkedFilesErrors
|
||||
|
||||
let ProjectFileAgent
|
||||
|
||||
export default ProjectFileAgent = {
|
||||
createLinkedFile(
|
||||
async function createLinkedFile(
|
||||
projectId,
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId,
|
||||
callback
|
||||
) {
|
||||
if (!ProjectFileAgent._canCreate(linkedFileData)) {
|
||||
return callback(new AccessDeniedError())
|
||||
historySource
|
||||
) {
|
||||
if (!_canCreate(linkedFileData)) {
|
||||
throw new AccessDeniedError()
|
||||
}
|
||||
return ProjectFileAgent._go(
|
||||
return await _go(
|
||||
projectId,
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId,
|
||||
callback
|
||||
historySource
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
refreshLinkedFile(
|
||||
async function refreshLinkedFile(
|
||||
projectId,
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId,
|
||||
callback
|
||||
) {
|
||||
return ProjectFileAgent._go(
|
||||
historySource
|
||||
) {
|
||||
return await _go(
|
||||
projectId,
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId,
|
||||
callback
|
||||
historySource
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
_prepare(projectId, linkedFileData, userId, callback) {
|
||||
if (callback == null) {
|
||||
callback = function () {}
|
||||
}
|
||||
return ProjectFileAgent._checkAuth(
|
||||
async function _go(
|
||||
projectId,
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId,
|
||||
(err, allowed) => {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
historySource
|
||||
) {
|
||||
linkedFileData = _sanitizeData(linkedFileData)
|
||||
await _prepare(projectId, linkedFileData, userId)
|
||||
const sourceProject =
|
||||
await LinkedFilesHandler.promises.getSourceProject(linkedFileData)
|
||||
const file = historySource
|
||||
? await _importFromHistory(
|
||||
projectId,
|
||||
sourceProject,
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
)
|
||||
: await _importFromProject(
|
||||
projectId,
|
||||
sourceProject,
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
)
|
||||
return file._id
|
||||
}
|
||||
|
||||
async function _prepare(projectId, linkedFileData, userId) {
|
||||
const allowed = await _checkAuth(projectId, linkedFileData, userId)
|
||||
if (!allowed) {
|
||||
return callback(new AccessDeniedError())
|
||||
throw new AccessDeniedError()
|
||||
}
|
||||
if (!ProjectFileAgent._validate(linkedFileData)) {
|
||||
return callback(new BadDataError())
|
||||
if (!_validate(linkedFileData)) {
|
||||
throw new BadDataError()
|
||||
}
|
||||
return callback(null, linkedFileData)
|
||||
}
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
_go(projectId, linkedFileData, name, parentFolderId, userId, callback) {
|
||||
linkedFileData = ProjectFileAgent._sanitizeData(linkedFileData)
|
||||
return ProjectFileAgent._prepare(
|
||||
/**
|
||||
* Read the source content from the source project's docs and files in mongo.
|
||||
*/
|
||||
async function _importFromProject(
|
||||
projectId,
|
||||
sourceProject,
|
||||
linkedFileData,
|
||||
userId,
|
||||
(err, linkedFileData) => {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
) {
|
||||
const sourceProjectId = sourceProject._id
|
||||
await DocumentUpdaterHandler.promises.flushProjectToMongo(sourceProjectId)
|
||||
|
||||
let entity, type
|
||||
try {
|
||||
;({ element: entity, type } =
|
||||
await ProjectLocator.promises.findElementByPath({
|
||||
project_id: sourceProjectId,
|
||||
path: linkedFileData.source_entity_path,
|
||||
exactCaseMatch: true,
|
||||
}))
|
||||
} catch (err) {
|
||||
if (err instanceof Errors.NotFoundError) {
|
||||
throw new SourceFileNotFoundError()
|
||||
}
|
||||
if (!ProjectFileAgent._validate(linkedFileData)) {
|
||||
return callback(new BadDataError())
|
||||
}
|
||||
return ProjectFileAgent._getEntity(
|
||||
linkedFileData,
|
||||
userId,
|
||||
(err, sourceProject, entity, type) => {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
throw err
|
||||
}
|
||||
|
||||
if (type === 'doc') {
|
||||
return DocstoreManager.getDoc(
|
||||
sourceProject._id,
|
||||
entity._id,
|
||||
function (err, lines) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
return LinkedFilesHandler.importContent(
|
||||
const { lines } = await DocstoreManager.promises.getDoc(
|
||||
sourceProjectId,
|
||||
entity._id
|
||||
)
|
||||
return await LinkedFilesHandler.promises.importContent(
|
||||
projectId,
|
||||
lines.join('\n'),
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId,
|
||||
function (err, file) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
return callback(null, file._id)
|
||||
}
|
||||
userId
|
||||
)
|
||||
}
|
||||
) // Created
|
||||
} else if (type === 'file') {
|
||||
return HistoryManager.requestBlob(
|
||||
const { stream } = await HistoryManager.promises.requestBlob(
|
||||
sourceProject.overleaf.history.id,
|
||||
entity.hash,
|
||||
'GET',
|
||||
function (err, result) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
return LinkedFilesHandler.importFromStream(
|
||||
entity.hash
|
||||
)
|
||||
return await LinkedFilesHandler.promises.importFromStream(
|
||||
projectId,
|
||||
result.stream,
|
||||
stream,
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId,
|
||||
function (err, file) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
return callback(null, file._id)
|
||||
}
|
||||
userId
|
||||
)
|
||||
}
|
||||
) // Created
|
||||
} else {
|
||||
return callback(new BadEntityTypeError())
|
||||
throw new BadEntityTypeError()
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
_getEntity(linkedFileData, currentUserId, callback) {
|
||||
if (callback == null) {
|
||||
callback = function () {}
|
||||
}
|
||||
callback = _.once(callback)
|
||||
const { source_entity_path: sourceEntityPath } = linkedFileData
|
||||
return ProjectFileAgent._getSourceProject(
|
||||
/**
|
||||
* Read the source content from the latest snapshot in the source project's
|
||||
* history, where docs and files are both plain files.
|
||||
*/
|
||||
async function _importFromHistory(
|
||||
projectId,
|
||||
sourceProject,
|
||||
linkedFileData,
|
||||
function (err, project) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
const sourceProjectId = project._id
|
||||
return DocumentUpdaterHandler.flushProjectToMongo(
|
||||
sourceProjectId,
|
||||
function (err) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
return ProjectLocator.findElementByPath(
|
||||
{
|
||||
project_id: sourceProjectId,
|
||||
path: sourceEntityPath,
|
||||
exactCaseMatch: true,
|
||||
},
|
||||
function (err, entity, type) {
|
||||
if (err != null) {
|
||||
if (err instanceof Errors.NotFoundError) {
|
||||
err = new SourceFileNotFoundError()
|
||||
}
|
||||
return callback(err)
|
||||
}
|
||||
return callback(null, project, entity, type)
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
},
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
) {
|
||||
const sourceProjectId = sourceProject._id
|
||||
const sourceHistoryId = sourceProject.overleaf.history.id
|
||||
|
||||
_sanitizeData(data) {
|
||||
// The snapshot only contains what has reached history, so push any pending
|
||||
// updates through document-updater and project-history first.
|
||||
await DocumentUpdaterHandler.promises.flushProjectToMongo(sourceProjectId)
|
||||
await HistoryManager.promises.flushProject(sourceProjectId)
|
||||
|
||||
const { snapshot } =
|
||||
await HistoryOTContentManager.getLatestSnapshot(sourceHistoryId)
|
||||
|
||||
// Paths in the project tree are absolute (/main.tex), history pathnames are
|
||||
// relative (main.tex).
|
||||
const pathname = linkedFileData.source_entity_path.replace(/^\//, '')
|
||||
const file = snapshot.getFile(pathname)
|
||||
if (!file) {
|
||||
if (
|
||||
snapshot
|
||||
.getFilePathnames()
|
||||
.some(other => other.startsWith(`${pathname}/`))
|
||||
) {
|
||||
// History has no folder entities, a folder is only visible as the prefix
|
||||
// of the pathnames below it.
|
||||
throw new BadEntityTypeError()
|
||||
}
|
||||
throw new SourceFileNotFoundError()
|
||||
}
|
||||
|
||||
// An eager load applies any edit operations that are not part of a blob yet.
|
||||
// It only reads a blob for an editable file, the other kinds resolve to
|
||||
// themselves.
|
||||
await file.load('eager', new HistoryBlobStore(sourceHistoryId))
|
||||
|
||||
const hash = file.getHash()
|
||||
if (hash) {
|
||||
// A file that is not editable is content-addressed by a single blob, so
|
||||
// stream it rather than holding it in memory.
|
||||
const { stream } = await HistoryManager.promises.requestBlob(
|
||||
sourceHistoryId,
|
||||
hash
|
||||
)
|
||||
return await LinkedFilesHandler.promises.importFromStream(
|
||||
projectId,
|
||||
stream,
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
)
|
||||
}
|
||||
|
||||
return await LinkedFilesHandler.promises.importContent(
|
||||
projectId,
|
||||
file.getContent({ filterTrackedDeletes: true }),
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
)
|
||||
}
|
||||
|
||||
function _sanitizeData(data) {
|
||||
return _.pick(
|
||||
data,
|
||||
'provider',
|
||||
@@ -221,50 +241,39 @@ export default ProjectFileAgent = {
|
||||
'source_entity_path',
|
||||
'importedAt'
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
_validate(data) {
|
||||
function _validate(data) {
|
||||
return (
|
||||
(data.source_project_id != null || data.v1_source_doc_id != null) &&
|
||||
data.source_entity_path != null
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
_canCreate(data) {
|
||||
function _canCreate(data) {
|
||||
// Don't allow creation of linked-files with v1 doc ids
|
||||
return data.v1_source_doc_id == null
|
||||
},
|
||||
}
|
||||
|
||||
_getSourceProject: LinkedFilesHandler.getSourceProject,
|
||||
|
||||
_checkAuth(projectId, data, currentUserId, callback) {
|
||||
if (callback == null) {
|
||||
callback = function () {}
|
||||
async function _checkAuth(projectId, data, currentUserId) {
|
||||
if (!_validate(data)) {
|
||||
throw new BadDataError()
|
||||
}
|
||||
callback = _.once(callback)
|
||||
if (!ProjectFileAgent._validate(data)) {
|
||||
return callback(new BadDataError())
|
||||
}
|
||||
return ProjectFileAgent._getSourceProject(data, function (err, project) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
return AuthorizationManager.canUserReadProject(
|
||||
const project = await LinkedFilesHandler.promises.getSourceProject(data)
|
||||
return await AuthorizationManager.promises.canUserReadProject(
|
||||
currentUserId,
|
||||
project._id,
|
||||
null,
|
||||
function (err, canRead) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
return callback(null, canRead)
|
||||
}
|
||||
null
|
||||
)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
ProjectFileAgent.promises = {
|
||||
createLinkedFile: promisify(ProjectFileAgent.createLinkedFile),
|
||||
refreshLinkedFile: promisify(ProjectFileAgent.refreshLinkedFile),
|
||||
export default {
|
||||
createLinkedFile: callbackify(createLinkedFile),
|
||||
refreshLinkedFile: callbackify(refreshLinkedFile),
|
||||
promises: { createLinkedFile, refreshLinkedFile },
|
||||
_sanitizeData,
|
||||
_validate,
|
||||
_canCreate,
|
||||
_getSourceProject: LinkedFilesHandler.getSourceProject,
|
||||
_checkAuth: callbackify(_checkAuth),
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ function createLinkedFile(
|
||||
name,
|
||||
parentFolderId,
|
||||
userId,
|
||||
_historySource,
|
||||
callback
|
||||
) {
|
||||
if (!ProjectFileAgent._canCreate(linkedFileData)) {
|
||||
@@ -70,6 +71,7 @@ function refreshLinkedFile(
|
||||
name,
|
||||
parentFolderId,
|
||||
userId,
|
||||
_historySource,
|
||||
callback
|
||||
) {
|
||||
_prepare(projectId, linkedFileData, userId, (err, linkedFileData) => {
|
||||
|
||||
@@ -14,7 +14,8 @@ async function createLinkedFile(
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
userId,
|
||||
_historySource
|
||||
) {
|
||||
logger.info(
|
||||
{ projectId, userId, url: linkedFileData.url },
|
||||
@@ -50,14 +51,16 @@ async function refreshLinkedFile(
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
userId,
|
||||
historySource
|
||||
) {
|
||||
return await createLinkedFile(
|
||||
projectId,
|
||||
linkedFileData,
|
||||
name,
|
||||
parentFolderId,
|
||||
userId
|
||||
userId,
|
||||
historySource
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ module.exports = base.mergeWith({
|
||||
|
||||
splitTestOverrides: {
|
||||
'sharing-updates-new-link': 'enabled', // routes in acceptance tests
|
||||
'linked-file-from-history': 'enabled', // read linked project files from history
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@@ -1,12 +1,19 @@
|
||||
import { expect } from 'chai'
|
||||
import crypto from 'node:crypto'
|
||||
import _ from 'lodash'
|
||||
import timekeeper from 'timekeeper'
|
||||
import Settings from '@overleaf/settings'
|
||||
import { CacheFlow } from 'cache-flow'
|
||||
import UserHelper from './helpers/User.mjs'
|
||||
import express from 'express'
|
||||
import { plainTextResponse } from '../../../app/src/infrastructure/Response.mjs'
|
||||
import { parseReq, z, handleValidationError } from '@overleaf/validation-tools'
|
||||
import { expectValidationErrorRaw } from '@overleaf/validation-tools/testUtils.js'
|
||||
import Features from '../../../app/src/infrastructure/Features.mjs'
|
||||
import { db } from '../../../app/src/infrastructure/mongodb.mjs'
|
||||
import MockDocstoreApiClass from './mocks/MockDocstoreApi.mjs'
|
||||
import MockProjectHistoryApiClass from './mocks/MockProjectHistoryApi.mjs'
|
||||
import MockV1HistoryApiClass from './mocks/MockV1HistoryApi.mjs'
|
||||
|
||||
const User = UserHelper.promises
|
||||
|
||||
@@ -31,6 +38,64 @@ LinkedUrlProxy.get('/', (req, res, next) => {
|
||||
})
|
||||
LinkedUrlProxy.use(handleValidationError)
|
||||
|
||||
// history-v1 addresses blobs by the git-style sha1 of their content.
|
||||
function blobHash(content) {
|
||||
const buffer = Buffer.from(content)
|
||||
return crypto
|
||||
.createHash('sha1')
|
||||
.update(`blob ${buffer.byteLength}\0`)
|
||||
.update(buffer)
|
||||
.digest('hex')
|
||||
}
|
||||
|
||||
// The source of a linked project file is gated behind the
|
||||
// `linked-file-from-history` split test. In saas, assignments come from the
|
||||
// database, so seed a fully rolled out "enabled" version.
|
||||
async function enableSourceSplitTest() {
|
||||
await db.splittests.updateOne(
|
||||
{ name: 'linked-file-from-history' },
|
||||
{
|
||||
$set: {
|
||||
versions: [
|
||||
{
|
||||
versionNumber: 1,
|
||||
createdAt: new Date(),
|
||||
active: true,
|
||||
analyticsEnabled: false,
|
||||
phase: 'release',
|
||||
variants: [
|
||||
{
|
||||
name: 'enabled',
|
||||
rolloutPercent: 100,
|
||||
rolloutStripes: [{ start: 0, end: 100 }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{ upsert: true }
|
||||
)
|
||||
await CacheFlow.reset('split-test')
|
||||
}
|
||||
|
||||
// The linked file endpoints have no query string of their own, they pick up the
|
||||
// split test override from the referer of the editor page.
|
||||
function editorReferer(projectId, variant) {
|
||||
return `${Settings.siteUrl}/project/${projectId}?linked-file-from-history=${variant}`
|
||||
}
|
||||
|
||||
// Both sources produce the same file, so the cases below run against each of
|
||||
// them. Outside saas the override in the referer has no effect and both cases
|
||||
// read from history, as set in splitTestOverrides.
|
||||
const SOURCE_VARIANTS = [
|
||||
{ description: 'with the source read from history', variant: 'enabled' },
|
||||
{
|
||||
description: 'with the source read from the project tree',
|
||||
variant: 'default',
|
||||
},
|
||||
]
|
||||
|
||||
describe('LinkedFiles', function () {
|
||||
before(function () {
|
||||
timekeeper.freeze(new Date())
|
||||
@@ -43,9 +108,19 @@ describe('LinkedFiles', function () {
|
||||
let projectOne, projectOneId, projectOneRootFolderId
|
||||
let projectTwo, projectTwoId, projectTwoRootFolderId
|
||||
const sourceDocName = 'test.txt'
|
||||
const sourceDocContent = 'source doc content'
|
||||
const sourceFileName = '1pixel.png'
|
||||
const sourceFolderName = 'subdir'
|
||||
let sourceDocId
|
||||
let owner
|
||||
|
||||
let MockDocstoreApi, MockProjectHistoryApi, MockV1HistoryApi
|
||||
before(function () {
|
||||
MockDocstoreApi = MockDocstoreApiClass.instance()
|
||||
MockProjectHistoryApi = MockProjectHistoryApiClass.instance()
|
||||
MockV1HistoryApi = MockV1HistoryApiClass.instance()
|
||||
})
|
||||
|
||||
let server
|
||||
before(function (done) {
|
||||
server = LinkedUrlProxy.listen(6543, done)
|
||||
@@ -60,6 +135,25 @@ describe('LinkedFiles', function () {
|
||||
})
|
||||
|
||||
describe('creating a project linked file', function () {
|
||||
let sourceHistoryId, sourceFileHash, snapshotFiles, snapshotChanges
|
||||
|
||||
// The latest chunk of the source project, as history-v1 serves it.
|
||||
function registerSourceSnapshot() {
|
||||
MockV1HistoryApi.addChunk(sourceHistoryId, {
|
||||
history: {
|
||||
snapshot: { files: snapshotFiles },
|
||||
changes: snapshotChanges,
|
||||
},
|
||||
startVersion: 0,
|
||||
})
|
||||
}
|
||||
|
||||
beforeEach(async function () {
|
||||
if (Features.hasFeature('saas')) {
|
||||
await enableSourceSplitTest()
|
||||
}
|
||||
})
|
||||
|
||||
beforeEach(async function () {
|
||||
projectOneId = await owner.createProject('plf-test-one', {
|
||||
template: 'blank',
|
||||
@@ -72,17 +166,70 @@ describe('LinkedFiles', function () {
|
||||
})
|
||||
projectTwo = await owner.getProject(projectTwoId)
|
||||
projectTwoRootFolderId = projectTwo.rootFolder[0]._id.toString()
|
||||
sourceHistoryId = projectTwo.overleaf.history.id.toString()
|
||||
|
||||
sourceDocId = await owner.createDocInProject(
|
||||
projectTwoId,
|
||||
projectTwoRootFolderId,
|
||||
sourceDocName
|
||||
)
|
||||
MockDocstoreApi.addDocument(projectTwoId, sourceDocId, {
|
||||
lines: [sourceDocContent],
|
||||
})
|
||||
await owner.createDocInProject(
|
||||
projectTwoId,
|
||||
projectTwoRootFolderId,
|
||||
'some-harmless-doc.txt'
|
||||
)
|
||||
|
||||
const { body: sourceFolder } = await owner.doRequest('post', {
|
||||
url: `/project/${projectTwoId}/folder`,
|
||||
json: {
|
||||
name: sourceFolderName,
|
||||
parent_folder_id: projectTwoRootFolderId,
|
||||
},
|
||||
})
|
||||
await owner.createDocInProject(
|
||||
projectTwoId,
|
||||
sourceFolder._id,
|
||||
'nested.txt'
|
||||
)
|
||||
|
||||
await owner.uploadExampleFileInProject(
|
||||
projectTwoId,
|
||||
projectTwoRootFolderId,
|
||||
sourceFileName
|
||||
)
|
||||
projectTwo = await owner.getProject(projectTwoId)
|
||||
sourceFileHash = _.find(
|
||||
projectTwo.rootFolder[0].fileRefs,
|
||||
file => file.name === sourceFileName
|
||||
).hash
|
||||
|
||||
// Docs live in the docstore, so their blob only exists in history.
|
||||
MockV1HistoryApi.addBlob(
|
||||
sourceHistoryId,
|
||||
blobHash(sourceDocContent),
|
||||
sourceDocContent
|
||||
)
|
||||
snapshotFiles = {
|
||||
'main.tex': { hash: blobHash(''), stringLength: 0 },
|
||||
[sourceDocName]: {
|
||||
hash: blobHash(sourceDocContent),
|
||||
stringLength: sourceDocContent.length,
|
||||
},
|
||||
[`${sourceFolderName}/nested.txt`]: {
|
||||
hash: blobHash(''),
|
||||
stringLength: 0,
|
||||
},
|
||||
[sourceFileName]: {
|
||||
hash: sourceFileHash,
|
||||
byteLength:
|
||||
MockV1HistoryApi.blobs[sourceHistoryId][sourceFileHash].byteLength,
|
||||
},
|
||||
}
|
||||
snapshotChanges = []
|
||||
registerSourceSnapshot()
|
||||
})
|
||||
|
||||
it('should produce a list of the users projects and their entities', async function () {
|
||||
@@ -112,18 +259,36 @@ describe('LinkedFiles', function () {
|
||||
expect(body).to.deep.equal({
|
||||
project_id: projectTwoId,
|
||||
entities: [
|
||||
{ path: `/${sourceFileName}`, type: 'file' },
|
||||
{ path: '/main.tex', type: 'doc' },
|
||||
{ path: '/some-harmless-doc.txt', type: 'doc' },
|
||||
{ path: '/test.txt', type: 'doc' },
|
||||
{ path: `/${sourceFolderName}/nested.txt`, type: 'doc' },
|
||||
{ path: `/${sourceDocName}`, type: 'doc' },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
for (const { description, variant } of SOURCE_VARIANTS) {
|
||||
describe(description, function () {
|
||||
function createLinkedFile(json) {
|
||||
return owner.doRequest('post', {
|
||||
url: `/project/${projectOneId}/linked_file`,
|
||||
json,
|
||||
headers: { referer: editorReferer(projectOneId, variant) },
|
||||
})
|
||||
}
|
||||
|
||||
function refreshLinkedFile(fileId) {
|
||||
return owner.doRequest('post', {
|
||||
url: `/project/${projectOneId}/linked_file/${fileId}/refresh`,
|
||||
json: true,
|
||||
headers: { referer: editorReferer(projectOneId, variant) },
|
||||
})
|
||||
}
|
||||
|
||||
it('should import a file and refresh it if there is no v1 id', async function () {
|
||||
// import the file from the source project
|
||||
let { response, body } = await owner.doRequest('post', {
|
||||
url: `/project/${projectOneId}/linked_file`,
|
||||
json: {
|
||||
let { response, body } = await createLinkedFile({
|
||||
name: 'test-link.txt',
|
||||
parent_folder_id: projectOneRootFolderId,
|
||||
provider: 'project_file',
|
||||
@@ -131,7 +296,6 @@ describe('LinkedFiles', function () {
|
||||
source_project_id: projectTwoId,
|
||||
source_entity_path: `/${sourceDocName}`,
|
||||
},
|
||||
},
|
||||
})
|
||||
expect(response.statusCode).to.equal(200)
|
||||
const existingFileId = body.new_file_id
|
||||
@@ -148,12 +312,14 @@ describe('LinkedFiles', function () {
|
||||
importedAt: new Date().toISOString(),
|
||||
})
|
||||
expect(firstFile.name).to.equal('test-link.txt')
|
||||
;({ body } = await owner.doRequest(
|
||||
'get',
|
||||
`/project/${projectOneId}/file/${existingFileId}`
|
||||
))
|
||||
expect(body).to.equal(sourceDocContent)
|
||||
|
||||
// refresh the file
|
||||
;({ response, body } = await owner.doRequest('post', {
|
||||
url: `/project/${projectOneId}/linked_file/${existingFileId}/refresh`,
|
||||
json: true,
|
||||
}))
|
||||
;({ response, body } = await refreshLinkedFile(existingFileId))
|
||||
expect(response.statusCode).to.equal(200)
|
||||
const newFileId = body.new_file_id
|
||||
expect(newFileId).to.exist
|
||||
@@ -165,9 +331,7 @@ describe('LinkedFiles', function () {
|
||||
expect(firstFile.name).to.equal('test-link.txt')
|
||||
|
||||
// should not work if there is a v1 id
|
||||
;({ response, body } = await owner.doRequest('post', {
|
||||
url: `/project/${projectOneId}/linked_file`,
|
||||
json: {
|
||||
;({ response, body } = await createLinkedFile({
|
||||
name: 'test-link-should-not-work.txt',
|
||||
parent_folder_id: projectOneRootFolderId,
|
||||
provider: 'project_file',
|
||||
@@ -175,7 +339,6 @@ describe('LinkedFiles', function () {
|
||||
v1_source_doc_id: 1234,
|
||||
source_entity_path: `/${sourceDocName}`,
|
||||
},
|
||||
},
|
||||
}))
|
||||
expect(response.statusCode).to.equal(403)
|
||||
expect(body).to.equal(
|
||||
@@ -183,9 +346,95 @@ describe('LinkedFiles', function () {
|
||||
)
|
||||
})
|
||||
|
||||
it('should import a binary file from the source project', async function () {
|
||||
const { response, body } = await createLinkedFile({
|
||||
name: 'test-link.png',
|
||||
parent_folder_id: projectOneRootFolderId,
|
||||
provider: 'project_file',
|
||||
data: {
|
||||
source_project_id: projectTwoId,
|
||||
source_entity_path: `/${sourceFileName}`,
|
||||
},
|
||||
})
|
||||
expect(response.statusCode).to.equal(200)
|
||||
|
||||
const updatedProjectOne = await owner.getProject(projectOneId)
|
||||
const file = _.find(
|
||||
updatedProjectOne.rootFolder[0].fileRefs,
|
||||
file => file.name === 'test-link.png'
|
||||
)
|
||||
expect(file._id.toString()).to.equal(body.new_file_id.toString())
|
||||
expect(file.hash).to.equal(sourceFileHash)
|
||||
})
|
||||
|
||||
it('should refuse to import a folder', async function () {
|
||||
const { response, body } = await createLinkedFile({
|
||||
name: 'test-link-folder.txt',
|
||||
parent_folder_id: projectOneRootFolderId,
|
||||
provider: 'project_file',
|
||||
data: {
|
||||
source_project_id: projectTwoId,
|
||||
source_entity_path: `/${sourceFolderName}`,
|
||||
},
|
||||
})
|
||||
expect(response.statusCode).to.equal(400)
|
||||
expect(body).to.equal('The file is the wrong type')
|
||||
})
|
||||
|
||||
it('should generate a proper error message when the source file has been deleted', async function () {
|
||||
// import the file from the source project
|
||||
let { response, body } = await owner.doRequest('post', {
|
||||
const { response: createResponse, body: createBody } =
|
||||
await createLinkedFile({
|
||||
name: 'test-link.txt',
|
||||
parent_folder_id: projectOneRootFolderId,
|
||||
provider: 'project_file',
|
||||
data: {
|
||||
source_project_id: projectTwoId,
|
||||
source_entity_path: `/${sourceDocName}`,
|
||||
},
|
||||
})
|
||||
expect(createResponse.statusCode).to.equal(200)
|
||||
const existingFileId = createBody.new_file_id
|
||||
expect(existingFileId).to.exist
|
||||
|
||||
// rename the source file
|
||||
await owner.renameItemInProject(
|
||||
projectTwoId,
|
||||
'doc',
|
||||
sourceDocId,
|
||||
'renamed-doc.txt'
|
||||
)
|
||||
// the rename reaches history as a move of the pathname
|
||||
snapshotFiles['renamed-doc.txt'] = snapshotFiles[sourceDocName]
|
||||
delete snapshotFiles[sourceDocName]
|
||||
registerSourceSnapshot()
|
||||
|
||||
// refresh the file
|
||||
const { response, body } = await refreshLinkedFile(existingFileId)
|
||||
expect(response.statusCode).to.equal(404)
|
||||
expect(body).to.equal('Source file not found')
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
it('should import edits that only reach history on flush', async function () {
|
||||
// Edits that are still in document-updater arrive in the snapshot as
|
||||
// operations on top of the last blob of the doc, once the source project
|
||||
// has been flushed.
|
||||
snapshotChanges = [
|
||||
{
|
||||
operations: [
|
||||
{
|
||||
pathname: sourceDocName,
|
||||
textOperation: ['edited ', sourceDocContent.length],
|
||||
},
|
||||
],
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
]
|
||||
registerSourceSnapshot()
|
||||
|
||||
const { response, body } = await owner.doRequest('post', {
|
||||
url: `/project/${projectOneId}/linked_file`,
|
||||
json: {
|
||||
name: 'test-link.txt',
|
||||
@@ -196,26 +445,16 @@ describe('LinkedFiles', function () {
|
||||
source_entity_path: `/${sourceDocName}`,
|
||||
},
|
||||
},
|
||||
headers: { referer: editorReferer(projectOneId, 'enabled') },
|
||||
})
|
||||
expect(response.statusCode).to.equal(200)
|
||||
const existingFileId = body.new_file_id
|
||||
expect(existingFileId).to.exist
|
||||
expect(MockProjectHistoryApi.flushedProjects).to.include(projectTwoId)
|
||||
|
||||
// rename the source file
|
||||
await owner.renameItemInProject(
|
||||
projectTwoId,
|
||||
'doc',
|
||||
sourceDocId,
|
||||
'renamed-doc.txt'
|
||||
const { body: content } = await owner.doRequest(
|
||||
'get',
|
||||
`/project/${projectOneId}/file/${body.new_file_id}`
|
||||
)
|
||||
|
||||
// refresh the file
|
||||
;({ response, body } = await owner.doRequest('post', {
|
||||
url: `/project/${projectOneId}/linked_file/${existingFileId}/refresh`,
|
||||
json: true,
|
||||
}))
|
||||
expect(response.statusCode).to.equal(404)
|
||||
expect(body).to.equal('Source file not found')
|
||||
expect(content).to.equal(`edited ${sourceDocContent}`)
|
||||
})
|
||||
|
||||
it('should reject a malformed file id when refreshing a linked file', async function () {
|
||||
|
||||
@@ -71,6 +71,7 @@ class MockProjectHistoryApi extends AbstractMockApi {
|
||||
this.labels = {}
|
||||
this.projectSnapshots = {}
|
||||
this.projectHistoryId = 1
|
||||
this.flushedProjects = []
|
||||
}
|
||||
|
||||
addOldFile(projectId, version, pathname, content) {
|
||||
@@ -208,6 +209,8 @@ class MockProjectHistoryApi extends AbstractMockApi {
|
||||
})
|
||||
|
||||
this.app.post('/project/:projectId/flush', (req, res) => {
|
||||
const { params } = parseReq(req, projectParamsSchema)
|
||||
this.flushedProjects.push(params.projectId.toString())
|
||||
res.sendStatus(200)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -56,6 +56,11 @@ const blobParamsSchema = z.object({
|
||||
}),
|
||||
})
|
||||
|
||||
// matches history-v1's getLatestHistory
|
||||
const latestHistoryParamsSchema = z.object({
|
||||
params: z.strictObject({ project_id: zz.projectHistoryId() }),
|
||||
})
|
||||
|
||||
// matches history-v1's copyProjectBlob query (params aren't read by this
|
||||
// mock's handler, so they're left out of the schema)
|
||||
const copyProjectBlobSchema = z.object({
|
||||
@@ -72,6 +77,18 @@ class MockV1HistoryApi extends AbstractMockApi {
|
||||
this.sentChunks = 0
|
||||
this.events = new EventEmitter()
|
||||
this.blobs = {}
|
||||
this.chunks = {}
|
||||
}
|
||||
|
||||
addBlob(historyId, hash, content) {
|
||||
if (!this.blobs[historyId]) {
|
||||
this.blobs[historyId] = {}
|
||||
}
|
||||
this.blobs[historyId][hash] = Buffer.from(content)
|
||||
}
|
||||
|
||||
addChunk(historyId, chunk) {
|
||||
this.chunks[historyId] = chunk
|
||||
}
|
||||
|
||||
computeBlobStats(historyId, blobHashes) {
|
||||
@@ -243,6 +260,16 @@ class MockV1HistoryApi extends AbstractMockApi {
|
||||
parseReq(req, copyProjectBlobSchema)
|
||||
res.sendStatus(204)
|
||||
})
|
||||
|
||||
this.app.get(
|
||||
'/api/projects/:project_id/latest/history',
|
||||
(req, res, next) => {
|
||||
const { params } = parseReq(req, latestHistoryParamsSchema)
|
||||
const chunk = this.chunks[params.project_id]
|
||||
if (!chunk) return res.sendStatus(404)
|
||||
res.json({ chunk })
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -54,6 +54,16 @@ describe('LinkedFilesController', function () {
|
||||
error: sinon.stub(),
|
||||
}
|
||||
ctx.settings = { enabledLinkedFileTypes: [] }
|
||||
ctx.SplitTestHandler = {
|
||||
promises: { featureFlagEnabled: sinon.stub().resolves(false) },
|
||||
}
|
||||
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/SplitTests/SplitTestHandler',
|
||||
() => ({
|
||||
default: ctx.SplitTestHandler,
|
||||
})
|
||||
)
|
||||
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/Authentication/SessionManager',
|
||||
@@ -153,7 +163,8 @@ describe('LinkedFilesController', function () {
|
||||
},
|
||||
ctx.fileName,
|
||||
ctx.parentFolderId,
|
||||
ctx.userId
|
||||
ctx.userId,
|
||||
false
|
||||
)
|
||||
resolve()
|
||||
},
|
||||
@@ -162,6 +173,34 @@ describe('LinkedFilesController', function () {
|
||||
})
|
||||
})
|
||||
|
||||
it('passes on the linked-file-from-history assignment', async function (ctx) {
|
||||
ctx.SplitTestHandler.promises.featureFlagEnabled.resolves(true)
|
||||
await new Promise((resolve, reject) => {
|
||||
ctx.next = sinon
|
||||
.stub()
|
||||
.callsFake(err =>
|
||||
reject(err || new Error('next called unexpectedly'))
|
||||
)
|
||||
ctx.res = {
|
||||
json: () => {
|
||||
expect(
|
||||
ctx.SplitTestHandler.promises.featureFlagEnabled
|
||||
).to.have.been.calledWith(
|
||||
ctx.req,
|
||||
ctx.res,
|
||||
'linked-file-from-history',
|
||||
{ includeReferer: true }
|
||||
)
|
||||
expect(
|
||||
ctx.Agent.promises.createLinkedFile.firstCall.args[5]
|
||||
).to.equal(true)
|
||||
resolve()
|
||||
},
|
||||
}
|
||||
ctx.LinkedFilesController.createLinkedFile(ctx.req, ctx.res, ctx.next)
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a mendeley group_id containing a path separator without calling the agent', async function (ctx) {
|
||||
setReqValidationModeForTests('enforce')
|
||||
ctx.req.body.provider = 'mendeley'
|
||||
@@ -245,7 +284,8 @@ describe('LinkedFilesController', function () {
|
||||
},
|
||||
ctx.fileName,
|
||||
'parent-folder-id',
|
||||
ctx.userId
|
||||
ctx.userId,
|
||||
false
|
||||
)
|
||||
resolve()
|
||||
},
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import sinon from 'sinon'
|
||||
import { Readable } from 'node:stream'
|
||||
|
||||
const modulePath =
|
||||
'../../../../app/src/Features/LinkedFiles/ProjectFileAgent.mjs'
|
||||
|
||||
const SOURCE_BLOB_HASH = 'a1b2c3d4e5f60718293a4b5c6d7e8f9012345678'
|
||||
const SOURCE_CONTENT = 'source content'
|
||||
|
||||
function rawChunk(files, changes = []) {
|
||||
return {
|
||||
chunk: {
|
||||
history: { snapshot: { files }, changes },
|
||||
startVersion: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
describe('ProjectFileAgent', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.projectId = 'project-id'
|
||||
ctx.fileName = 'linked-file.txt'
|
||||
ctx.parentFolderId = 'parent-folder-id'
|
||||
ctx.userId = 'user-id'
|
||||
ctx.linkedFileData = {
|
||||
provider: 'project_file',
|
||||
source_project_id: 'source-project-id',
|
||||
source_entity_path: '/test.txt',
|
||||
}
|
||||
ctx.sourceProject = {
|
||||
_id: 'source-project-id',
|
||||
name: 'source project',
|
||||
overleaf: { history: { id: 'source-history-id' } },
|
||||
}
|
||||
ctx.file = { _id: 'new-file-id' }
|
||||
ctx.blobStoreHashes = []
|
||||
|
||||
ctx.AuthorizationManager = {
|
||||
promises: { canUserReadProject: sinon.stub().resolves(true) },
|
||||
}
|
||||
ctx.ProjectLocator = {
|
||||
promises: { findElementByPath: sinon.stub() },
|
||||
}
|
||||
ctx.DocstoreManager = {
|
||||
promises: {
|
||||
getDoc: sinon.stub().resolves({ lines: ['source', 'content'] }),
|
||||
},
|
||||
}
|
||||
ctx.DocumentUpdaterHandler = {
|
||||
promises: { flushProjectToMongo: sinon.stub().resolves() },
|
||||
}
|
||||
ctx.LinkedFilesHandler = {
|
||||
getSourceProject: sinon.stub(),
|
||||
promises: {
|
||||
getSourceProject: sinon.stub().resolves(ctx.sourceProject),
|
||||
importContent: sinon.stub().resolves(ctx.file),
|
||||
importFromStream: sinon.stub().resolves(ctx.file),
|
||||
},
|
||||
}
|
||||
ctx.HistoryManager = {
|
||||
promises: {
|
||||
flushProject: sinon.stub().resolves(),
|
||||
requestBlob: sinon
|
||||
.stub()
|
||||
.resolves({ stream: Readable.from([SOURCE_CONTENT]) }),
|
||||
getLatestHistoryWithHistoryId: sinon.stub(),
|
||||
},
|
||||
}
|
||||
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/Authorization/AuthorizationManager.mjs',
|
||||
() => ({ default: ctx.AuthorizationManager })
|
||||
)
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/Project/ProjectLocator.mjs',
|
||||
() => ({
|
||||
default: ctx.ProjectLocator,
|
||||
})
|
||||
)
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/Docstore/DocstoreManager.mjs',
|
||||
() => ({ default: ctx.DocstoreManager })
|
||||
)
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.mjs',
|
||||
() => ({ default: ctx.DocumentUpdaterHandler })
|
||||
)
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/LinkedFiles/LinkedFilesHandler.mjs',
|
||||
() => ({ default: ctx.LinkedFilesHandler })
|
||||
)
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/History/HistoryManager.mjs',
|
||||
() => ({ default: ctx.HistoryManager })
|
||||
)
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/History/HistoryBlobStore.mjs',
|
||||
() => ({
|
||||
default: class {
|
||||
constructor(historyId) {
|
||||
ctx.blobStoreHistoryId = historyId
|
||||
}
|
||||
|
||||
async getString(hash) {
|
||||
ctx.blobStoreHashes.push(hash)
|
||||
return SOURCE_CONTENT
|
||||
}
|
||||
},
|
||||
})
|
||||
)
|
||||
// Import the error classes from the same module graph as the module under
|
||||
// test, so that instanceof checks on both sides agree.
|
||||
ctx.Errors = (
|
||||
await import('../../../../app/src/Features/Errors/Errors.js')
|
||||
).default
|
||||
ctx.LinkedFilesErrors = (
|
||||
await import('../../../../app/src/Features/LinkedFiles/LinkedFilesErrors.mjs')
|
||||
).default
|
||||
ctx.ProjectFileAgent = (await import(modulePath)).default
|
||||
ctx.createLinkedFile = () =>
|
||||
ctx.ProjectFileAgent.promises.createLinkedFile(
|
||||
ctx.projectId,
|
||||
ctx.linkedFileData,
|
||||
ctx.fileName,
|
||||
ctx.parentFolderId,
|
||||
ctx.userId,
|
||||
ctx.historySource
|
||||
)
|
||||
})
|
||||
|
||||
describe('_canCreate', function () {
|
||||
it('rejects v1 source doc ids', function (ctx) {
|
||||
expect(
|
||||
ctx.ProjectFileAgent._canCreate({ v1_source_doc_id: 1234 })
|
||||
).to.equal(false)
|
||||
expect(
|
||||
ctx.ProjectFileAgent._canCreate({ source_project_id: 'id' })
|
||||
).to.equal(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('createLinkedFile', function () {
|
||||
it('refuses to create a linked file with a v1 source doc id', async function (ctx) {
|
||||
ctx.linkedFileData = {
|
||||
provider: 'project_file',
|
||||
v1_source_doc_id: 1234,
|
||||
source_entity_path: '/test.txt',
|
||||
}
|
||||
await expect(ctx.createLinkedFile()).to.be.rejectedWith(
|
||||
ctx.LinkedFilesErrors.AccessDeniedError
|
||||
)
|
||||
})
|
||||
|
||||
it('refuses to read a source project the user cannot access', async function (ctx) {
|
||||
ctx.AuthorizationManager.promises.canUserReadProject.resolves(false)
|
||||
await expect(ctx.createLinkedFile()).to.be.rejectedWith(
|
||||
ctx.LinkedFilesErrors.AccessDeniedError
|
||||
)
|
||||
})
|
||||
|
||||
it('refuses to import without a source entity path', async function (ctx) {
|
||||
delete ctx.linkedFileData.source_entity_path
|
||||
await expect(ctx.createLinkedFile()).to.be.rejectedWith(
|
||||
ctx.LinkedFilesErrors.BadDataError
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('with the source in the project tree', function () {
|
||||
beforeEach(function (ctx) {
|
||||
ctx.historySource = false
|
||||
})
|
||||
|
||||
it('imports the content of a doc', async function (ctx) {
|
||||
ctx.ProjectLocator.promises.findElementByPath.resolves({
|
||||
element: { _id: 'source-doc-id' },
|
||||
type: 'doc',
|
||||
})
|
||||
|
||||
expect(await ctx.createLinkedFile()).to.equal('new-file-id')
|
||||
|
||||
expect(
|
||||
ctx.DocumentUpdaterHandler.promises.flushProjectToMongo
|
||||
).to.have.been.calledWith('source-project-id')
|
||||
expect(
|
||||
ctx.ProjectLocator.promises.findElementByPath
|
||||
).to.have.been.calledWith({
|
||||
project_id: 'source-project-id',
|
||||
path: '/test.txt',
|
||||
exactCaseMatch: true,
|
||||
})
|
||||
expect(
|
||||
ctx.LinkedFilesHandler.promises.importContent
|
||||
).to.have.been.calledWith(
|
||||
ctx.projectId,
|
||||
'source\ncontent',
|
||||
ctx.linkedFileData,
|
||||
ctx.fileName,
|
||||
ctx.parentFolderId,
|
||||
ctx.userId
|
||||
)
|
||||
})
|
||||
|
||||
it('imports the content of a file from its blob', async function (ctx) {
|
||||
ctx.ProjectLocator.promises.findElementByPath.resolves({
|
||||
element: { _id: 'source-file-id', hash: SOURCE_BLOB_HASH },
|
||||
type: 'file',
|
||||
})
|
||||
|
||||
expect(await ctx.createLinkedFile()).to.equal('new-file-id')
|
||||
|
||||
expect(ctx.HistoryManager.promises.requestBlob).to.have.been.calledWith(
|
||||
'source-history-id',
|
||||
SOURCE_BLOB_HASH
|
||||
)
|
||||
expect(ctx.LinkedFilesHandler.promises.importFromStream).to.have.been
|
||||
.called
|
||||
})
|
||||
|
||||
it('reports a missing source file', async function (ctx) {
|
||||
ctx.ProjectLocator.promises.findElementByPath.rejects(
|
||||
new ctx.Errors.NotFoundError()
|
||||
)
|
||||
await expect(ctx.createLinkedFile()).to.be.rejectedWith(
|
||||
ctx.LinkedFilesErrors.SourceFileNotFoundError
|
||||
)
|
||||
})
|
||||
|
||||
it('reports a source path that is a folder', async function (ctx) {
|
||||
ctx.ProjectLocator.promises.findElementByPath.resolves({
|
||||
element: { _id: 'source-folder-id' },
|
||||
type: 'folder',
|
||||
})
|
||||
await expect(ctx.createLinkedFile()).to.be.rejectedWith(
|
||||
ctx.LinkedFilesErrors.BadEntityTypeError
|
||||
)
|
||||
})
|
||||
|
||||
it('does not read from history', async function (ctx) {
|
||||
ctx.ProjectLocator.promises.findElementByPath.resolves({
|
||||
element: { _id: 'source-doc-id' },
|
||||
type: 'doc',
|
||||
})
|
||||
await ctx.createLinkedFile()
|
||||
|
||||
expect(ctx.HistoryManager.promises.flushProject).to.not.have.been.called
|
||||
expect(ctx.HistoryManager.promises.getLatestHistoryWithHistoryId).to.not
|
||||
.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('with the source in the history snapshot', function () {
|
||||
beforeEach(function (ctx) {
|
||||
ctx.historySource = true
|
||||
ctx.HistoryManager.promises.getLatestHistoryWithHistoryId.resolves(
|
||||
rawChunk({
|
||||
'test.txt': {
|
||||
hash: SOURCE_BLOB_HASH,
|
||||
stringLength: SOURCE_CONTENT.length,
|
||||
},
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('flushes the source project into history before reading it', async function (ctx) {
|
||||
await ctx.createLinkedFile()
|
||||
|
||||
expect(
|
||||
ctx.DocumentUpdaterHandler.promises.flushProjectToMongo
|
||||
).to.have.been.calledWith('source-project-id')
|
||||
expect(ctx.HistoryManager.promises.flushProject).to.have.been.calledWith(
|
||||
'source-project-id'
|
||||
)
|
||||
expect(
|
||||
ctx.HistoryManager.promises.getLatestHistoryWithHistoryId
|
||||
).to.have.been.calledWith('source-history-id')
|
||||
expect(
|
||||
ctx.HistoryManager.promises.flushProject
|
||||
).to.have.been.calledBefore(
|
||||
ctx.HistoryManager.promises.getLatestHistoryWithHistoryId
|
||||
)
|
||||
})
|
||||
|
||||
it('does not look up the source path in the project tree', async function (ctx) {
|
||||
await ctx.createLinkedFile()
|
||||
|
||||
expect(ctx.ProjectLocator.promises.findElementByPath).to.not.have.been
|
||||
.called
|
||||
})
|
||||
|
||||
it('imports the content of an editable file', async function (ctx) {
|
||||
expect(await ctx.createLinkedFile()).to.equal('new-file-id')
|
||||
|
||||
expect(ctx.blobStoreHistoryId).to.equal('source-history-id')
|
||||
expect(ctx.blobStoreHashes).to.deep.equal([SOURCE_BLOB_HASH])
|
||||
expect(
|
||||
ctx.LinkedFilesHandler.promises.importContent
|
||||
).to.have.been.calledWith(
|
||||
ctx.projectId,
|
||||
SOURCE_CONTENT,
|
||||
ctx.linkedFileData,
|
||||
ctx.fileName,
|
||||
ctx.parentFolderId,
|
||||
ctx.userId
|
||||
)
|
||||
expect(ctx.HistoryManager.promises.requestBlob).to.not.have.been.called
|
||||
})
|
||||
|
||||
it('streams the blob of a file that is not editable', async function (ctx) {
|
||||
ctx.HistoryManager.promises.getLatestHistoryWithHistoryId.resolves(
|
||||
rawChunk({
|
||||
'test.txt': {
|
||||
hash: SOURCE_BLOB_HASH,
|
||||
byteLength: SOURCE_CONTENT.length,
|
||||
},
|
||||
})
|
||||
)
|
||||
|
||||
expect(await ctx.createLinkedFile()).to.equal('new-file-id')
|
||||
|
||||
expect(ctx.HistoryManager.promises.requestBlob).to.have.been.calledWith(
|
||||
'source-history-id',
|
||||
SOURCE_BLOB_HASH
|
||||
)
|
||||
expect(ctx.LinkedFilesHandler.promises.importFromStream).to.have.been
|
||||
.called
|
||||
expect(ctx.LinkedFilesHandler.promises.importContent).to.not.have.been
|
||||
.called
|
||||
expect(ctx.blobStoreHashes).to.deep.equal([])
|
||||
})
|
||||
|
||||
it('imports the content of a file with pending edits', async function (ctx) {
|
||||
ctx.HistoryManager.promises.getLatestHistoryWithHistoryId.resolves(
|
||||
rawChunk(
|
||||
{
|
||||
'test.txt': {
|
||||
hash: SOURCE_BLOB_HASH,
|
||||
stringLength: SOURCE_CONTENT.length,
|
||||
},
|
||||
},
|
||||
[
|
||||
{
|
||||
operations: [
|
||||
{
|
||||
pathname: 'test.txt',
|
||||
textOperation: ['edited ', SOURCE_CONTENT.length],
|
||||
},
|
||||
],
|
||||
timestamp: '2026-08-11T09:00:00.000Z',
|
||||
},
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
expect(await ctx.createLinkedFile()).to.equal('new-file-id')
|
||||
|
||||
expect(ctx.blobStoreHistoryId).to.equal('source-history-id')
|
||||
expect(ctx.blobStoreHashes).to.deep.equal([SOURCE_BLOB_HASH])
|
||||
expect(
|
||||
ctx.LinkedFilesHandler.promises.importContent
|
||||
).to.have.been.calledWith(
|
||||
ctx.projectId,
|
||||
`edited ${SOURCE_CONTENT}`,
|
||||
ctx.linkedFileData,
|
||||
ctx.fileName,
|
||||
ctx.parentFolderId,
|
||||
ctx.userId
|
||||
)
|
||||
expect(ctx.HistoryManager.promises.requestBlob).to.not.have.been.called
|
||||
})
|
||||
|
||||
it('accepts a source path without a leading slash', async function (ctx) {
|
||||
ctx.linkedFileData.source_entity_path = 'test.txt'
|
||||
expect(await ctx.createLinkedFile()).to.equal('new-file-id')
|
||||
})
|
||||
|
||||
it('reports a missing source file', async function (ctx) {
|
||||
ctx.linkedFileData.source_entity_path = '/other.txt'
|
||||
await expect(ctx.createLinkedFile()).to.be.rejectedWith(
|
||||
ctx.LinkedFilesErrors.SourceFileNotFoundError
|
||||
)
|
||||
})
|
||||
|
||||
it('reports a source path that is a folder', async function (ctx) {
|
||||
ctx.HistoryManager.promises.getLatestHistoryWithHistoryId.resolves(
|
||||
rawChunk({
|
||||
'folder/test.txt': {
|
||||
hash: SOURCE_BLOB_HASH,
|
||||
stringLength: SOURCE_CONTENT.length,
|
||||
},
|
||||
})
|
||||
)
|
||||
ctx.linkedFileData.source_entity_path = '/folder'
|
||||
await expect(ctx.createLinkedFile()).to.be.rejectedWith(
|
||||
ctx.LinkedFilesErrors.BadEntityTypeError
|
||||
)
|
||||
})
|
||||
|
||||
it('reads the snapshot when refreshing too', async function (ctx) {
|
||||
expect(
|
||||
await ctx.ProjectFileAgent.promises.refreshLinkedFile(
|
||||
ctx.projectId,
|
||||
ctx.linkedFileData,
|
||||
ctx.fileName,
|
||||
ctx.parentFolderId,
|
||||
ctx.userId,
|
||||
true
|
||||
)
|
||||
).to.equal('new-file-id')
|
||||
|
||||
expect(ctx.HistoryManager.promises.flushProject).to.have.been.calledWith(
|
||||
'source-project-id'
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -7,6 +7,7 @@ type LinkedFileAgent = {
|
||||
name: string,
|
||||
parentFolderId: string,
|
||||
userId: string,
|
||||
historySource: boolean,
|
||||
callback: () => void
|
||||
) => void
|
||||
refreshLinkedFile: (
|
||||
@@ -15,6 +16,7 @@ type LinkedFileAgent = {
|
||||
name: string,
|
||||
parentFolderId: string,
|
||||
userId: string,
|
||||
historySource: boolean,
|
||||
callback: () => void
|
||||
) => void
|
||||
promises: {
|
||||
@@ -23,14 +25,16 @@ type LinkedFileAgent = {
|
||||
linkedFileData: object,
|
||||
name: string,
|
||||
parentFolderId: string,
|
||||
userId: string
|
||||
userId: string,
|
||||
historySource: boolean
|
||||
) => Promise<any>
|
||||
refreshLinkedFile: (
|
||||
projectId: string,
|
||||
linkedFileData: object,
|
||||
name: string,
|
||||
parentFolderId: string,
|
||||
userId: string
|
||||
userId: string,
|
||||
historySource: boolean
|
||||
) => Promise<any>
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user