diff --git a/services/web/app/src/Features/History/HistoryBlobStore.mjs b/services/web/app/src/Features/History/HistoryBlobStore.mjs new file mode 100644 index 0000000000..fd6736f845 --- /dev/null +++ b/services/web/app/src/Features/History/HistoryBlobStore.mjs @@ -0,0 +1,40 @@ +import { text } from 'node:stream/consumers' +import HistoryManager from './HistoryManager.mjs' + +/** + * The blob store of a project's history, as overleaf-editor-core wants it. + * + * Reading covers what File#load asks for, so a snapshot's files can be loaded + * eagerly here -- needed for the ones with pending edit operations, whose content + * is not in a single blob. + */ +class HistoryBlobStore { + /** + * @param {string} historyId + */ + constructor(historyId) { + this.historyId = historyId + } + + /** + * @param {string} hash + * @return {Promise} + */ + async getString(hash) { + const { stream } = await HistoryManager.promises.requestBlob( + this.historyId, + hash + ) + return await text(stream) + } + + /** + * @param {string} hash + * @return {Promise} + */ + async getObject(hash) { + return JSON.parse(await this.getString(hash)) + } +} + +export default HistoryBlobStore diff --git a/services/web/app/src/Features/History/HistoryOTContentManager.mjs b/services/web/app/src/Features/History/HistoryOTContentManager.mjs new file mode 100644 index 0000000000..5783765e18 --- /dev/null +++ b/services/web/app/src/Features/History/HistoryOTContentManager.mjs @@ -0,0 +1,68 @@ +import { setTimeout } from 'node:timers/promises' +import { Chunk } from 'overleaf-editor-core' +import logger from '@overleaf/logger' +import Metrics from '@overleaf/metrics' +import HistoryManager from './HistoryManager.mjs' + +/** + * @import { Snapshot } from 'overleaf-editor-core' + */ + +const READ_ATTEMPTS = 3 +const READ_RETRY_DELAY_MS = 100 + +/** + * Read from history, repeating the request if it fails. + * + * Retried on any failure, an error status included: history answering 500 to one + * request says no more about the next one than a connection that was dropped + * does. + * + * @template T + * @param {string} metricName + * @param {string} msg - logged with the error on each retry + * @param {object} info - logged with the error on each retry + * @param {() => Promise} fn + * @return {Promise} + */ +async function withRetries(metricName, msg, info, fn) { + for (let attempt = 1; ; attempt++) { + try { + return await fn() + } catch (err) { + if (attempt === READ_ATTEMPTS) throw err + Metrics.inc(metricName) + logger.warn({ err, attempt, ...info }, `${msg}, retrying`) + await setTimeout(attempt * READ_RETRY_DELAY_MS) + } + } +} + +/** + * Load the project's latest chunk. + * + * @param {string} historyId + * @return {Promise} + */ +async function fetchLatestChunk(historyId) { + const { chunk } = await withRetries( + 'history_ot_chunk_fetch_retry', + 'failed to get the latest history chunk', + { historyId }, + () => HistoryManager.promises.getLatestHistoryWithHistoryId(historyId) + ) + return Chunk.fromRaw(chunk) +} + +/** + * The project's latest snapshot, and the version it is of. + * + * @param {string} historyId + * @return {Promise<{snapshot: Snapshot, version: number}>} + */ +export async function getLatestSnapshot(historyId) { + const chunk = await fetchLatestChunk(historyId) + const snapshot = chunk.getSnapshot() + snapshot.applyAll(chunk.getChanges()) + return { snapshot, version: chunk.getEndVersion() } +} diff --git a/services/web/app/src/Features/LinkedFiles/LinkedFilesController.mjs b/services/web/app/src/Features/LinkedFiles/LinkedFilesController.mjs index daf01bd782..c0f982b0e5 100644 --- a/services/web/app/src/Features/LinkedFiles/LinkedFilesController.mjs +++ b/services/web/app/src/Features/LinkedFiles/LinkedFilesController.mjs @@ -22,6 +22,7 @@ import { TooManyFilesError, } from '../Errors/Errors.js' import Modules from '../../infrastructure/Modules.mjs' +import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs' import { plainTextResponse } from '../../infrastructure/Response.mjs' import { z, zz, parseReq } from '../../infrastructure/Validation.mjs' import EditorRealTimeController from '../Editor/EditorRealTimeController.mjs' @@ -177,13 +178,21 @@ async function createLinkedFile(req, res, next) { data.provider = provider data.importedAt = new Date().toISOString() + const historySource = await SplitTestHandler.promises.featureFlagEnabled( + req, + res, + 'linked-file-from-history', + { includeReferer: true } + ) + try { const newFileId = await Agent.promises.createLinkedFile( projectId, data, name, parentFolderId, - userId + userId, + historySource ) if (name.endsWith('.bib')) { AnalyticsManager.recordEventForSession(req.session, 'linked-bib-file', { @@ -229,6 +238,14 @@ async function refreshLinkedFile(req, res, next) { } linkedFileData.importedAt = new Date().toISOString() + + const historySource = await SplitTestHandler.promises.featureFlagEnabled( + req, + res, + 'linked-file-from-history', + { includeReferer: true } + ) + let newFileId try { newFileId = await Agent.promises.refreshLinkedFile( @@ -236,7 +253,8 @@ async function refreshLinkedFile(req, res, next) { linkedFileData, name, parentFolderId, - userId + userId, + historySource ) } catch (err) { return LinkedFilesController.handleError(err, req, res, next) diff --git a/services/web/app/src/Features/LinkedFiles/ProjectFileAgent.mjs b/services/web/app/src/Features/LinkedFiles/ProjectFileAgent.mjs index b71db46148..b1aa5b154c 100644 --- a/services/web/app/src/Features/LinkedFiles/ProjectFileAgent.mjs +++ b/services/web/app/src/Features/LinkedFiles/ProjectFileAgent.mjs @@ -1,15 +1,3 @@ -/* eslint-disable - n/handle-callback-err, - max-len, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ import AuthorizationManager from '../Authorization/AuthorizationManager.mjs' import ProjectLocator from '../Project/ProjectLocator.mjs' import DocstoreManager from '../Docstore/DocstoreManager.mjs' @@ -17,8 +5,10 @@ import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mj import _ from 'lodash' import LinkedFilesHandler from './LinkedFilesHandler.mjs' import LinkedFilesErrors from './LinkedFilesErrors.mjs' -import { promisify } from '@overleaf/promise-utils' +import { callbackify } from '@overleaf/promise-utils' import HistoryManager from '../History/HistoryManager.mjs' +import HistoryBlobStore from '../History/HistoryBlobStore.mjs' +import * as HistoryOTContentManager from '../History/HistoryOTContentManager.mjs' import Errors from '../Errors/Errors.js' const { @@ -28,243 +18,262 @@ const { SourceFileNotFoundError, } = LinkedFilesErrors -let ProjectFileAgent - -export default ProjectFileAgent = { - createLinkedFile( +async function createLinkedFile( + projectId, + linkedFileData, + name, + parentFolderId, + userId, + historySource +) { + if (!_canCreate(linkedFileData)) { + throw new AccessDeniedError() + } + return await _go( projectId, linkedFileData, name, parentFolderId, userId, - callback - ) { - if (!ProjectFileAgent._canCreate(linkedFileData)) { - return callback(new AccessDeniedError()) - } - return ProjectFileAgent._go( - projectId, - linkedFileData, - name, - parentFolderId, - userId, - callback - ) - }, + historySource + ) +} - refreshLinkedFile( +async function refreshLinkedFile( + projectId, + linkedFileData, + name, + parentFolderId, + userId, + historySource +) { + return await _go( projectId, linkedFileData, name, parentFolderId, userId, - callback - ) { - return ProjectFileAgent._go( - projectId, - linkedFileData, - name, - parentFolderId, - userId, - callback - ) - }, + historySource + ) +} - _prepare(projectId, linkedFileData, userId, callback) { - if (callback == null) { - callback = function () {} - } - return ProjectFileAgent._checkAuth( - projectId, - linkedFileData, - userId, - (err, allowed) => { - if (err != null) { - return callback(err) - } - if (!allowed) { - return callback(new AccessDeniedError()) - } - if (!ProjectFileAgent._validate(linkedFileData)) { - return callback(new BadDataError()) - } - return callback(null, linkedFileData) - } - ) - }, - - _go(projectId, linkedFileData, name, parentFolderId, userId, callback) { - linkedFileData = ProjectFileAgent._sanitizeData(linkedFileData) - return ProjectFileAgent._prepare( - projectId, - linkedFileData, - userId, - (err, linkedFileData) => { - if (err != null) { - return callback(err) - } - if (!ProjectFileAgent._validate(linkedFileData)) { - return callback(new BadDataError()) - } - return ProjectFileAgent._getEntity( - linkedFileData, - userId, - (err, sourceProject, entity, type) => { - if (err != null) { - return callback(err) - } - if (type === 'doc') { - return DocstoreManager.getDoc( - sourceProject._id, - entity._id, - function (err, lines) { - if (err != null) { - return callback(err) - } - return LinkedFilesHandler.importContent( - projectId, - lines.join('\n'), - linkedFileData, - name, - parentFolderId, - userId, - function (err, file) { - if (err != null) { - return callback(err) - } - return callback(null, file._id) - } - ) - } - ) // Created - } else if (type === 'file') { - return HistoryManager.requestBlob( - sourceProject.overleaf.history.id, - entity.hash, - 'GET', - function (err, result) { - if (err != null) { - return callback(err) - } - return LinkedFilesHandler.importFromStream( - projectId, - result.stream, - linkedFileData, - name, - parentFolderId, - userId, - function (err, file) { - if (err != null) { - return callback(err) - } - return callback(null, file._id) - } - ) - } - ) // Created - } else { - return callback(new BadEntityTypeError()) - } - } - ) - } - ) - }, - - _getEntity(linkedFileData, currentUserId, callback) { - if (callback == null) { - callback = function () {} - } - callback = _.once(callback) - const { source_entity_path: sourceEntityPath } = linkedFileData - return ProjectFileAgent._getSourceProject( - linkedFileData, - function (err, project) { - if (err != null) { - return callback(err) - } - const sourceProjectId = project._id - return DocumentUpdaterHandler.flushProjectToMongo( - sourceProjectId, - function (err) { - if (err != null) { - return callback(err) - } - return ProjectLocator.findElementByPath( - { - project_id: sourceProjectId, - path: sourceEntityPath, - exactCaseMatch: true, - }, - function (err, entity, type) { - if (err != null) { - if (err instanceof Errors.NotFoundError) { - err = new SourceFileNotFoundError() - } - return callback(err) - } - return callback(null, project, entity, type) - } - ) - } - ) - } - ) - }, - - _sanitizeData(data) { - return _.pick( - data, - 'provider', - 'source_project_id', - 'v1_source_doc_id', - 'source_entity_path', - 'importedAt' - ) - }, - - _validate(data) { - return ( - (data.source_project_id != null || data.v1_source_doc_id != null) && - data.source_entity_path != null - ) - }, - - _canCreate(data) { - // Don't allow creation of linked-files with v1 doc ids - return data.v1_source_doc_id == null - }, - - _getSourceProject: LinkedFilesHandler.getSourceProject, - - _checkAuth(projectId, data, currentUserId, callback) { - if (callback == null) { - callback = function () {} - } - callback = _.once(callback) - if (!ProjectFileAgent._validate(data)) { - return callback(new BadDataError()) - } - return ProjectFileAgent._getSourceProject(data, function (err, project) { - if (err != null) { - return callback(err) - } - return AuthorizationManager.canUserReadProject( - currentUserId, - project._id, - null, - function (err, canRead) { - if (err != null) { - return callback(err) - } - return callback(null, canRead) - } +async function _go( + projectId, + linkedFileData, + name, + parentFolderId, + userId, + historySource +) { + linkedFileData = _sanitizeData(linkedFileData) + await _prepare(projectId, linkedFileData, userId) + const sourceProject = + await LinkedFilesHandler.promises.getSourceProject(linkedFileData) + const file = historySource + ? await _importFromHistory( + projectId, + sourceProject, + linkedFileData, + name, + parentFolderId, + userId ) - }) - }, + : await _importFromProject( + projectId, + sourceProject, + linkedFileData, + name, + parentFolderId, + userId + ) + return file._id } -ProjectFileAgent.promises = { - createLinkedFile: promisify(ProjectFileAgent.createLinkedFile), - refreshLinkedFile: promisify(ProjectFileAgent.refreshLinkedFile), +async function _prepare(projectId, linkedFileData, userId) { + const allowed = await _checkAuth(projectId, linkedFileData, userId) + if (!allowed) { + throw new AccessDeniedError() + } + if (!_validate(linkedFileData)) { + throw new BadDataError() + } +} + +/** + * Read the source content from the source project's docs and files in mongo. + */ +async function _importFromProject( + projectId, + sourceProject, + linkedFileData, + name, + parentFolderId, + userId +) { + const sourceProjectId = sourceProject._id + await DocumentUpdaterHandler.promises.flushProjectToMongo(sourceProjectId) + + let entity, type + try { + ;({ element: entity, type } = + await ProjectLocator.promises.findElementByPath({ + project_id: sourceProjectId, + path: linkedFileData.source_entity_path, + exactCaseMatch: true, + })) + } catch (err) { + if (err instanceof Errors.NotFoundError) { + throw new SourceFileNotFoundError() + } + throw err + } + + if (type === 'doc') { + const { lines } = await DocstoreManager.promises.getDoc( + sourceProjectId, + entity._id + ) + return await LinkedFilesHandler.promises.importContent( + projectId, + lines.join('\n'), + linkedFileData, + name, + parentFolderId, + userId + ) + } else if (type === 'file') { + const { stream } = await HistoryManager.promises.requestBlob( + sourceProject.overleaf.history.id, + entity.hash + ) + return await LinkedFilesHandler.promises.importFromStream( + projectId, + stream, + linkedFileData, + name, + parentFolderId, + userId + ) + } else { + throw new BadEntityTypeError() + } +} + +/** + * Read the source content from the latest snapshot in the source project's + * history, where docs and files are both plain files. + */ +async function _importFromHistory( + projectId, + sourceProject, + linkedFileData, + name, + parentFolderId, + userId +) { + const sourceProjectId = sourceProject._id + const sourceHistoryId = sourceProject.overleaf.history.id + + // The snapshot only contains what has reached history, so push any pending + // updates through document-updater and project-history first. + await DocumentUpdaterHandler.promises.flushProjectToMongo(sourceProjectId) + await HistoryManager.promises.flushProject(sourceProjectId) + + const { snapshot } = + await HistoryOTContentManager.getLatestSnapshot(sourceHistoryId) + + // Paths in the project tree are absolute (/main.tex), history pathnames are + // relative (main.tex). + const pathname = linkedFileData.source_entity_path.replace(/^\//, '') + const file = snapshot.getFile(pathname) + if (!file) { + if ( + snapshot + .getFilePathnames() + .some(other => other.startsWith(`${pathname}/`)) + ) { + // History has no folder entities, a folder is only visible as the prefix + // of the pathnames below it. + throw new BadEntityTypeError() + } + throw new SourceFileNotFoundError() + } + + // An eager load applies any edit operations that are not part of a blob yet. + // It only reads a blob for an editable file, the other kinds resolve to + // themselves. + await file.load('eager', new HistoryBlobStore(sourceHistoryId)) + + const hash = file.getHash() + if (hash) { + // A file that is not editable is content-addressed by a single blob, so + // stream it rather than holding it in memory. + const { stream } = await HistoryManager.promises.requestBlob( + sourceHistoryId, + hash + ) + return await LinkedFilesHandler.promises.importFromStream( + projectId, + stream, + linkedFileData, + name, + parentFolderId, + userId + ) + } + + return await LinkedFilesHandler.promises.importContent( + projectId, + file.getContent({ filterTrackedDeletes: true }), + linkedFileData, + name, + parentFolderId, + userId + ) +} + +function _sanitizeData(data) { + return _.pick( + data, + 'provider', + 'source_project_id', + 'v1_source_doc_id', + 'source_entity_path', + 'importedAt' + ) +} + +function _validate(data) { + return ( + (data.source_project_id != null || data.v1_source_doc_id != null) && + data.source_entity_path != null + ) +} + +function _canCreate(data) { + // Don't allow creation of linked-files with v1 doc ids + return data.v1_source_doc_id == null +} + +async function _checkAuth(projectId, data, currentUserId) { + if (!_validate(data)) { + throw new BadDataError() + } + const project = await LinkedFilesHandler.promises.getSourceProject(data) + return await AuthorizationManager.promises.canUserReadProject( + currentUserId, + project._id, + null + ) +} + +export default { + createLinkedFile: callbackify(createLinkedFile), + refreshLinkedFile: callbackify(refreshLinkedFile), + promises: { createLinkedFile, refreshLinkedFile }, + _sanitizeData, + _validate, + _canCreate, + _getSourceProject: LinkedFilesHandler.getSourceProject, + _checkAuth: callbackify(_checkAuth), } diff --git a/services/web/app/src/Features/LinkedFiles/ProjectOutputFileAgent.mjs b/services/web/app/src/Features/LinkedFiles/ProjectOutputFileAgent.mjs index 024212dcc5..45cacec953 100644 --- a/services/web/app/src/Features/LinkedFiles/ProjectOutputFileAgent.mjs +++ b/services/web/app/src/Features/LinkedFiles/ProjectOutputFileAgent.mjs @@ -32,6 +32,7 @@ function createLinkedFile( name, parentFolderId, userId, + _historySource, callback ) { if (!ProjectFileAgent._canCreate(linkedFileData)) { @@ -70,6 +71,7 @@ function refreshLinkedFile( name, parentFolderId, userId, + _historySource, callback ) { _prepare(projectId, linkedFileData, userId, (err, linkedFileData) => { diff --git a/services/web/app/src/Features/LinkedFiles/UrlAgent.mjs b/services/web/app/src/Features/LinkedFiles/UrlAgent.mjs index 732724ad77..ffe9c04944 100644 --- a/services/web/app/src/Features/LinkedFiles/UrlAgent.mjs +++ b/services/web/app/src/Features/LinkedFiles/UrlAgent.mjs @@ -14,7 +14,8 @@ async function createLinkedFile( linkedFileData, name, parentFolderId, - userId + userId, + _historySource ) { logger.info( { projectId, userId, url: linkedFileData.url }, @@ -50,14 +51,16 @@ async function refreshLinkedFile( linkedFileData, name, parentFolderId, - userId + userId, + historySource ) { return await createLinkedFile( projectId, linkedFileData, name, parentFolderId, - userId + userId, + historySource ) } diff --git a/services/web/test/acceptance/config/settings.test.server-ce.js b/services/web/test/acceptance/config/settings.test.server-ce.js index ffeef58a28..32e6851bb6 100644 --- a/services/web/test/acceptance/config/settings.test.server-ce.js +++ b/services/web/test/acceptance/config/settings.test.server-ce.js @@ -8,6 +8,7 @@ module.exports = base.mergeWith({ splitTestOverrides: { 'sharing-updates-new-link': 'enabled', // routes in acceptance tests + 'linked-file-from-history': 'enabled', // read linked project files from history }, }) diff --git a/services/web/test/acceptance/src/LinkedFilesTests.mjs b/services/web/test/acceptance/src/LinkedFilesTests.mjs index 14fb5faa87..e00b070d81 100644 --- a/services/web/test/acceptance/src/LinkedFilesTests.mjs +++ b/services/web/test/acceptance/src/LinkedFilesTests.mjs @@ -1,12 +1,19 @@ import { expect } from 'chai' +import crypto from 'node:crypto' import _ from 'lodash' import timekeeper from 'timekeeper' import Settings from '@overleaf/settings' +import { CacheFlow } from 'cache-flow' import UserHelper from './helpers/User.mjs' import express from 'express' import { plainTextResponse } from '../../../app/src/infrastructure/Response.mjs' import { parseReq, z, handleValidationError } from '@overleaf/validation-tools' import { expectValidationErrorRaw } from '@overleaf/validation-tools/testUtils.js' +import Features from '../../../app/src/infrastructure/Features.mjs' +import { db } from '../../../app/src/infrastructure/mongodb.mjs' +import MockDocstoreApiClass from './mocks/MockDocstoreApi.mjs' +import MockProjectHistoryApiClass from './mocks/MockProjectHistoryApi.mjs' +import MockV1HistoryApiClass from './mocks/MockV1HistoryApi.mjs' const User = UserHelper.promises @@ -31,6 +38,64 @@ LinkedUrlProxy.get('/', (req, res, next) => { }) LinkedUrlProxy.use(handleValidationError) +// history-v1 addresses blobs by the git-style sha1 of their content. +function blobHash(content) { + const buffer = Buffer.from(content) + return crypto + .createHash('sha1') + .update(`blob ${buffer.byteLength}\0`) + .update(buffer) + .digest('hex') +} + +// The source of a linked project file is gated behind the +// `linked-file-from-history` split test. In saas, assignments come from the +// database, so seed a fully rolled out "enabled" version. +async function enableSourceSplitTest() { + await db.splittests.updateOne( + { name: 'linked-file-from-history' }, + { + $set: { + versions: [ + { + versionNumber: 1, + createdAt: new Date(), + active: true, + analyticsEnabled: false, + phase: 'release', + variants: [ + { + name: 'enabled', + rolloutPercent: 100, + rolloutStripes: [{ start: 0, end: 100 }], + }, + ], + }, + ], + }, + }, + { upsert: true } + ) + await CacheFlow.reset('split-test') +} + +// The linked file endpoints have no query string of their own, they pick up the +// split test override from the referer of the editor page. +function editorReferer(projectId, variant) { + return `${Settings.siteUrl}/project/${projectId}?linked-file-from-history=${variant}` +} + +// Both sources produce the same file, so the cases below run against each of +// them. Outside saas the override in the referer has no effect and both cases +// read from history, as set in splitTestOverrides. +const SOURCE_VARIANTS = [ + { description: 'with the source read from history', variant: 'enabled' }, + { + description: 'with the source read from the project tree', + variant: 'default', + }, +] + describe('LinkedFiles', function () { before(function () { timekeeper.freeze(new Date()) @@ -43,9 +108,19 @@ describe('LinkedFiles', function () { let projectOne, projectOneId, projectOneRootFolderId let projectTwo, projectTwoId, projectTwoRootFolderId const sourceDocName = 'test.txt' + const sourceDocContent = 'source doc content' + const sourceFileName = '1pixel.png' + const sourceFolderName = 'subdir' let sourceDocId let owner + let MockDocstoreApi, MockProjectHistoryApi, MockV1HistoryApi + before(function () { + MockDocstoreApi = MockDocstoreApiClass.instance() + MockProjectHistoryApi = MockProjectHistoryApiClass.instance() + MockV1HistoryApi = MockV1HistoryApiClass.instance() + }) + let server before(function (done) { server = LinkedUrlProxy.listen(6543, done) @@ -60,6 +135,25 @@ describe('LinkedFiles', function () { }) describe('creating a project linked file', function () { + let sourceHistoryId, sourceFileHash, snapshotFiles, snapshotChanges + + // The latest chunk of the source project, as history-v1 serves it. + function registerSourceSnapshot() { + MockV1HistoryApi.addChunk(sourceHistoryId, { + history: { + snapshot: { files: snapshotFiles }, + changes: snapshotChanges, + }, + startVersion: 0, + }) + } + + beforeEach(async function () { + if (Features.hasFeature('saas')) { + await enableSourceSplitTest() + } + }) + beforeEach(async function () { projectOneId = await owner.createProject('plf-test-one', { template: 'blank', @@ -72,17 +166,70 @@ describe('LinkedFiles', function () { }) projectTwo = await owner.getProject(projectTwoId) projectTwoRootFolderId = projectTwo.rootFolder[0]._id.toString() + sourceHistoryId = projectTwo.overleaf.history.id.toString() sourceDocId = await owner.createDocInProject( projectTwoId, projectTwoRootFolderId, sourceDocName ) + MockDocstoreApi.addDocument(projectTwoId, sourceDocId, { + lines: [sourceDocContent], + }) await owner.createDocInProject( projectTwoId, projectTwoRootFolderId, 'some-harmless-doc.txt' ) + + const { body: sourceFolder } = await owner.doRequest('post', { + url: `/project/${projectTwoId}/folder`, + json: { + name: sourceFolderName, + parent_folder_id: projectTwoRootFolderId, + }, + }) + await owner.createDocInProject( + projectTwoId, + sourceFolder._id, + 'nested.txt' + ) + + await owner.uploadExampleFileInProject( + projectTwoId, + projectTwoRootFolderId, + sourceFileName + ) + projectTwo = await owner.getProject(projectTwoId) + sourceFileHash = _.find( + projectTwo.rootFolder[0].fileRefs, + file => file.name === sourceFileName + ).hash + + // Docs live in the docstore, so their blob only exists in history. + MockV1HistoryApi.addBlob( + sourceHistoryId, + blobHash(sourceDocContent), + sourceDocContent + ) + snapshotFiles = { + 'main.tex': { hash: blobHash(''), stringLength: 0 }, + [sourceDocName]: { + hash: blobHash(sourceDocContent), + stringLength: sourceDocContent.length, + }, + [`${sourceFolderName}/nested.txt`]: { + hash: blobHash(''), + stringLength: 0, + }, + [sourceFileName]: { + hash: sourceFileHash, + byteLength: + MockV1HistoryApi.blobs[sourceHistoryId][sourceFileHash].byteLength, + }, + } + snapshotChanges = [] + registerSourceSnapshot() }) it('should produce a list of the users projects and their entities', async function () { @@ -112,16 +259,182 @@ describe('LinkedFiles', function () { expect(body).to.deep.equal({ project_id: projectTwoId, entities: [ + { path: `/${sourceFileName}`, type: 'file' }, { path: '/main.tex', type: 'doc' }, { path: '/some-harmless-doc.txt', type: 'doc' }, - { path: '/test.txt', type: 'doc' }, + { path: `/${sourceFolderName}/nested.txt`, type: 'doc' }, + { path: `/${sourceDocName}`, type: 'doc' }, ], }) }) - it('should import a file and refresh it if there is no v1 id', async function () { - // import the file from the source project - let { response, body } = await owner.doRequest('post', { + for (const { description, variant } of SOURCE_VARIANTS) { + describe(description, function () { + function createLinkedFile(json) { + return owner.doRequest('post', { + url: `/project/${projectOneId}/linked_file`, + json, + headers: { referer: editorReferer(projectOneId, variant) }, + }) + } + + function refreshLinkedFile(fileId) { + return owner.doRequest('post', { + url: `/project/${projectOneId}/linked_file/${fileId}/refresh`, + json: true, + headers: { referer: editorReferer(projectOneId, variant) }, + }) + } + + it('should import a file and refresh it if there is no v1 id', async function () { + // import the file from the source project + let { response, body } = await createLinkedFile({ + name: 'test-link.txt', + parent_folder_id: projectOneRootFolderId, + provider: 'project_file', + data: { + source_project_id: projectTwoId, + source_entity_path: `/${sourceDocName}`, + }, + }) + expect(response.statusCode).to.equal(200) + const existingFileId = body.new_file_id + expect(existingFileId).to.exist + + let updatedProjectOne = await owner.getProject(projectOneId) + + let firstFile = updatedProjectOne.rootFolder[0].fileRefs[0] + expect(firstFile._id.toString()).to.equal(existingFileId.toString()) + expect(firstFile.linkedFileData).to.deep.equal({ + provider: 'project_file', + source_project_id: projectTwoId, + source_entity_path: `/${sourceDocName}`, + importedAt: new Date().toISOString(), + }) + expect(firstFile.name).to.equal('test-link.txt') + ;({ body } = await owner.doRequest( + 'get', + `/project/${projectOneId}/file/${existingFileId}` + )) + expect(body).to.equal(sourceDocContent) + + // refresh the file + ;({ response, body } = await refreshLinkedFile(existingFileId)) + expect(response.statusCode).to.equal(200) + const newFileId = body.new_file_id + expect(newFileId).to.exist + expect(newFileId).to.not.equal(existingFileId) + + updatedProjectOne = await owner.getProject(projectOneId) + firstFile = updatedProjectOne.rootFolder[0].fileRefs[0] + expect(firstFile._id.toString()).to.equal(newFileId.toString()) + expect(firstFile.name).to.equal('test-link.txt') + + // should not work if there is a v1 id + ;({ response, body } = await createLinkedFile({ + name: 'test-link-should-not-work.txt', + parent_folder_id: projectOneRootFolderId, + provider: 'project_file', + data: { + v1_source_doc_id: 1234, + source_entity_path: `/${sourceDocName}`, + }, + })) + expect(response.statusCode).to.equal(403) + expect(body).to.equal( + 'The project that contains this file is not shared with you' + ) + }) + + it('should import a binary file from the source project', async function () { + const { response, body } = await createLinkedFile({ + name: 'test-link.png', + parent_folder_id: projectOneRootFolderId, + provider: 'project_file', + data: { + source_project_id: projectTwoId, + source_entity_path: `/${sourceFileName}`, + }, + }) + expect(response.statusCode).to.equal(200) + + const updatedProjectOne = await owner.getProject(projectOneId) + const file = _.find( + updatedProjectOne.rootFolder[0].fileRefs, + file => file.name === 'test-link.png' + ) + expect(file._id.toString()).to.equal(body.new_file_id.toString()) + expect(file.hash).to.equal(sourceFileHash) + }) + + it('should refuse to import a folder', async function () { + const { response, body } = await createLinkedFile({ + name: 'test-link-folder.txt', + parent_folder_id: projectOneRootFolderId, + provider: 'project_file', + data: { + source_project_id: projectTwoId, + source_entity_path: `/${sourceFolderName}`, + }, + }) + expect(response.statusCode).to.equal(400) + expect(body).to.equal('The file is the wrong type') + }) + + it('should generate a proper error message when the source file has been deleted', async function () { + // import the file from the source project + const { response: createResponse, body: createBody } = + await createLinkedFile({ + name: 'test-link.txt', + parent_folder_id: projectOneRootFolderId, + provider: 'project_file', + data: { + source_project_id: projectTwoId, + source_entity_path: `/${sourceDocName}`, + }, + }) + expect(createResponse.statusCode).to.equal(200) + const existingFileId = createBody.new_file_id + expect(existingFileId).to.exist + + // rename the source file + await owner.renameItemInProject( + projectTwoId, + 'doc', + sourceDocId, + 'renamed-doc.txt' + ) + // the rename reaches history as a move of the pathname + snapshotFiles['renamed-doc.txt'] = snapshotFiles[sourceDocName] + delete snapshotFiles[sourceDocName] + registerSourceSnapshot() + + // refresh the file + const { response, body } = await refreshLinkedFile(existingFileId) + expect(response.statusCode).to.equal(404) + expect(body).to.equal('Source file not found') + }) + }) + } + + it('should import edits that only reach history on flush', async function () { + // Edits that are still in document-updater arrive in the snapshot as + // operations on top of the last blob of the doc, once the source project + // has been flushed. + snapshotChanges = [ + { + operations: [ + { + pathname: sourceDocName, + textOperation: ['edited ', sourceDocContent.length], + }, + ], + timestamp: new Date().toISOString(), + }, + ] + registerSourceSnapshot() + + const { response, body } = await owner.doRequest('post', { url: `/project/${projectOneId}/linked_file`, json: { name: 'test-link.txt', @@ -132,90 +445,16 @@ describe('LinkedFiles', function () { source_entity_path: `/${sourceDocName}`, }, }, + headers: { referer: editorReferer(projectOneId, 'enabled') }, }) expect(response.statusCode).to.equal(200) - const existingFileId = body.new_file_id - expect(existingFileId).to.exist + expect(MockProjectHistoryApi.flushedProjects).to.include(projectTwoId) - let updatedProjectOne = await owner.getProject(projectOneId) - - let firstFile = updatedProjectOne.rootFolder[0].fileRefs[0] - expect(firstFile._id.toString()).to.equal(existingFileId.toString()) - expect(firstFile.linkedFileData).to.deep.equal({ - provider: 'project_file', - source_project_id: projectTwoId, - source_entity_path: `/${sourceDocName}`, - importedAt: new Date().toISOString(), - }) - expect(firstFile.name).to.equal('test-link.txt') - - // refresh the file - ;({ response, body } = await owner.doRequest('post', { - url: `/project/${projectOneId}/linked_file/${existingFileId}/refresh`, - json: true, - })) - expect(response.statusCode).to.equal(200) - const newFileId = body.new_file_id - expect(newFileId).to.exist - expect(newFileId).to.not.equal(existingFileId) - - updatedProjectOne = await owner.getProject(projectOneId) - firstFile = updatedProjectOne.rootFolder[0].fileRefs[0] - expect(firstFile._id.toString()).to.equal(newFileId.toString()) - expect(firstFile.name).to.equal('test-link.txt') - - // should not work if there is a v1 id - ;({ response, body } = await owner.doRequest('post', { - url: `/project/${projectOneId}/linked_file`, - json: { - name: 'test-link-should-not-work.txt', - parent_folder_id: projectOneRootFolderId, - provider: 'project_file', - data: { - v1_source_doc_id: 1234, - source_entity_path: `/${sourceDocName}`, - }, - }, - })) - expect(response.statusCode).to.equal(403) - expect(body).to.equal( - 'The project that contains this file is not shared with you' + const { body: content } = await owner.doRequest( + 'get', + `/project/${projectOneId}/file/${body.new_file_id}` ) - }) - - it('should generate a proper error message when the source file has been deleted', async function () { - // import the file from the source project - let { response, body } = await owner.doRequest('post', { - url: `/project/${projectOneId}/linked_file`, - json: { - name: 'test-link.txt', - parent_folder_id: projectOneRootFolderId, - provider: 'project_file', - data: { - source_project_id: projectTwoId, - source_entity_path: `/${sourceDocName}`, - }, - }, - }) - expect(response.statusCode).to.equal(200) - const existingFileId = body.new_file_id - expect(existingFileId).to.exist - - // rename the source file - await owner.renameItemInProject( - projectTwoId, - 'doc', - sourceDocId, - 'renamed-doc.txt' - ) - - // refresh the file - ;({ response, body } = await owner.doRequest('post', { - url: `/project/${projectOneId}/linked_file/${existingFileId}/refresh`, - json: true, - })) - expect(response.statusCode).to.equal(404) - expect(body).to.equal('Source file not found') + expect(content).to.equal(`edited ${sourceDocContent}`) }) it('should reject a malformed file id when refreshing a linked file', async function () { diff --git a/services/web/test/acceptance/src/mocks/MockProjectHistoryApi.mjs b/services/web/test/acceptance/src/mocks/MockProjectHistoryApi.mjs index a700e43245..4e9b387fc6 100644 --- a/services/web/test/acceptance/src/mocks/MockProjectHistoryApi.mjs +++ b/services/web/test/acceptance/src/mocks/MockProjectHistoryApi.mjs @@ -71,6 +71,7 @@ class MockProjectHistoryApi extends AbstractMockApi { this.labels = {} this.projectSnapshots = {} this.projectHistoryId = 1 + this.flushedProjects = [] } addOldFile(projectId, version, pathname, content) { @@ -208,6 +209,8 @@ class MockProjectHistoryApi extends AbstractMockApi { }) this.app.post('/project/:projectId/flush', (req, res) => { + const { params } = parseReq(req, projectParamsSchema) + this.flushedProjects.push(params.projectId.toString()) res.sendStatus(200) }) } diff --git a/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs b/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs index b309bbc37d..af21be8fac 100644 --- a/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs +++ b/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs @@ -56,6 +56,11 @@ const blobParamsSchema = z.object({ }), }) +// matches history-v1's getLatestHistory +const latestHistoryParamsSchema = z.object({ + params: z.strictObject({ project_id: zz.projectHistoryId() }), +}) + // matches history-v1's copyProjectBlob query (params aren't read by this // mock's handler, so they're left out of the schema) const copyProjectBlobSchema = z.object({ @@ -72,6 +77,18 @@ class MockV1HistoryApi extends AbstractMockApi { this.sentChunks = 0 this.events = new EventEmitter() this.blobs = {} + this.chunks = {} + } + + addBlob(historyId, hash, content) { + if (!this.blobs[historyId]) { + this.blobs[historyId] = {} + } + this.blobs[historyId][hash] = Buffer.from(content) + } + + addChunk(historyId, chunk) { + this.chunks[historyId] = chunk } computeBlobStats(historyId, blobHashes) { @@ -243,6 +260,16 @@ class MockV1HistoryApi extends AbstractMockApi { parseReq(req, copyProjectBlobSchema) res.sendStatus(204) }) + + this.app.get( + '/api/projects/:project_id/latest/history', + (req, res, next) => { + const { params } = parseReq(req, latestHistoryParamsSchema) + const chunk = this.chunks[params.project_id] + if (!chunk) return res.sendStatus(404) + res.json({ chunk }) + } + ) } } diff --git a/services/web/test/unit/src/LinkedFiles/LinkedFilesController.test.mjs b/services/web/test/unit/src/LinkedFiles/LinkedFilesController.test.mjs index d8cd4077ee..9d12416e41 100644 --- a/services/web/test/unit/src/LinkedFiles/LinkedFilesController.test.mjs +++ b/services/web/test/unit/src/LinkedFiles/LinkedFilesController.test.mjs @@ -54,6 +54,16 @@ describe('LinkedFilesController', function () { error: sinon.stub(), } ctx.settings = { enabledLinkedFileTypes: [] } + ctx.SplitTestHandler = { + promises: { featureFlagEnabled: sinon.stub().resolves(false) }, + } + + vi.doMock( + '../../../../app/src/Features/SplitTests/SplitTestHandler', + () => ({ + default: ctx.SplitTestHandler, + }) + ) vi.doMock( '../../../../app/src/Features/Authentication/SessionManager', @@ -153,7 +163,8 @@ describe('LinkedFilesController', function () { }, ctx.fileName, ctx.parentFolderId, - ctx.userId + ctx.userId, + false ) resolve() }, @@ -162,6 +173,34 @@ describe('LinkedFilesController', function () { }) }) + it('passes on the linked-file-from-history assignment', async function (ctx) { + ctx.SplitTestHandler.promises.featureFlagEnabled.resolves(true) + await new Promise((resolve, reject) => { + ctx.next = sinon + .stub() + .callsFake(err => + reject(err || new Error('next called unexpectedly')) + ) + ctx.res = { + json: () => { + expect( + ctx.SplitTestHandler.promises.featureFlagEnabled + ).to.have.been.calledWith( + ctx.req, + ctx.res, + 'linked-file-from-history', + { includeReferer: true } + ) + expect( + ctx.Agent.promises.createLinkedFile.firstCall.args[5] + ).to.equal(true) + resolve() + }, + } + ctx.LinkedFilesController.createLinkedFile(ctx.req, ctx.res, ctx.next) + }) + }) + it('rejects a mendeley group_id containing a path separator without calling the agent', async function (ctx) { setReqValidationModeForTests('enforce') ctx.req.body.provider = 'mendeley' @@ -245,7 +284,8 @@ describe('LinkedFilesController', function () { }, ctx.fileName, 'parent-folder-id', - ctx.userId + ctx.userId, + false ) resolve() }, diff --git a/services/web/test/unit/src/LinkedFiles/ProjectFileAgent.test.mjs b/services/web/test/unit/src/LinkedFiles/ProjectFileAgent.test.mjs new file mode 100644 index 0000000000..195df6af60 --- /dev/null +++ b/services/web/test/unit/src/LinkedFiles/ProjectFileAgent.test.mjs @@ -0,0 +1,417 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import sinon from 'sinon' +import { Readable } from 'node:stream' + +const modulePath = + '../../../../app/src/Features/LinkedFiles/ProjectFileAgent.mjs' + +const SOURCE_BLOB_HASH = 'a1b2c3d4e5f60718293a4b5c6d7e8f9012345678' +const SOURCE_CONTENT = 'source content' + +function rawChunk(files, changes = []) { + return { + chunk: { + history: { snapshot: { files }, changes }, + startVersion: 0, + }, + } +} + +describe('ProjectFileAgent', function () { + beforeEach(async function (ctx) { + ctx.projectId = 'project-id' + ctx.fileName = 'linked-file.txt' + ctx.parentFolderId = 'parent-folder-id' + ctx.userId = 'user-id' + ctx.linkedFileData = { + provider: 'project_file', + source_project_id: 'source-project-id', + source_entity_path: '/test.txt', + } + ctx.sourceProject = { + _id: 'source-project-id', + name: 'source project', + overleaf: { history: { id: 'source-history-id' } }, + } + ctx.file = { _id: 'new-file-id' } + ctx.blobStoreHashes = [] + + ctx.AuthorizationManager = { + promises: { canUserReadProject: sinon.stub().resolves(true) }, + } + ctx.ProjectLocator = { + promises: { findElementByPath: sinon.stub() }, + } + ctx.DocstoreManager = { + promises: { + getDoc: sinon.stub().resolves({ lines: ['source', 'content'] }), + }, + } + ctx.DocumentUpdaterHandler = { + promises: { flushProjectToMongo: sinon.stub().resolves() }, + } + ctx.LinkedFilesHandler = { + getSourceProject: sinon.stub(), + promises: { + getSourceProject: sinon.stub().resolves(ctx.sourceProject), + importContent: sinon.stub().resolves(ctx.file), + importFromStream: sinon.stub().resolves(ctx.file), + }, + } + ctx.HistoryManager = { + promises: { + flushProject: sinon.stub().resolves(), + requestBlob: sinon + .stub() + .resolves({ stream: Readable.from([SOURCE_CONTENT]) }), + getLatestHistoryWithHistoryId: sinon.stub(), + }, + } + + vi.doMock( + '../../../../app/src/Features/Authorization/AuthorizationManager.mjs', + () => ({ default: ctx.AuthorizationManager }) + ) + vi.doMock( + '../../../../app/src/Features/Project/ProjectLocator.mjs', + () => ({ + default: ctx.ProjectLocator, + }) + ) + vi.doMock( + '../../../../app/src/Features/Docstore/DocstoreManager.mjs', + () => ({ default: ctx.DocstoreManager }) + ) + vi.doMock( + '../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.mjs', + () => ({ default: ctx.DocumentUpdaterHandler }) + ) + vi.doMock( + '../../../../app/src/Features/LinkedFiles/LinkedFilesHandler.mjs', + () => ({ default: ctx.LinkedFilesHandler }) + ) + vi.doMock( + '../../../../app/src/Features/History/HistoryManager.mjs', + () => ({ default: ctx.HistoryManager }) + ) + vi.doMock( + '../../../../app/src/Features/History/HistoryBlobStore.mjs', + () => ({ + default: class { + constructor(historyId) { + ctx.blobStoreHistoryId = historyId + } + + async getString(hash) { + ctx.blobStoreHashes.push(hash) + return SOURCE_CONTENT + } + }, + }) + ) + // Import the error classes from the same module graph as the module under + // test, so that instanceof checks on both sides agree. + ctx.Errors = ( + await import('../../../../app/src/Features/Errors/Errors.js') + ).default + ctx.LinkedFilesErrors = ( + await import('../../../../app/src/Features/LinkedFiles/LinkedFilesErrors.mjs') + ).default + ctx.ProjectFileAgent = (await import(modulePath)).default + ctx.createLinkedFile = () => + ctx.ProjectFileAgent.promises.createLinkedFile( + ctx.projectId, + ctx.linkedFileData, + ctx.fileName, + ctx.parentFolderId, + ctx.userId, + ctx.historySource + ) + }) + + describe('_canCreate', function () { + it('rejects v1 source doc ids', function (ctx) { + expect( + ctx.ProjectFileAgent._canCreate({ v1_source_doc_id: 1234 }) + ).to.equal(false) + expect( + ctx.ProjectFileAgent._canCreate({ source_project_id: 'id' }) + ).to.equal(true) + }) + }) + + describe('createLinkedFile', function () { + it('refuses to create a linked file with a v1 source doc id', async function (ctx) { + ctx.linkedFileData = { + provider: 'project_file', + v1_source_doc_id: 1234, + source_entity_path: '/test.txt', + } + await expect(ctx.createLinkedFile()).to.be.rejectedWith( + ctx.LinkedFilesErrors.AccessDeniedError + ) + }) + + it('refuses to read a source project the user cannot access', async function (ctx) { + ctx.AuthorizationManager.promises.canUserReadProject.resolves(false) + await expect(ctx.createLinkedFile()).to.be.rejectedWith( + ctx.LinkedFilesErrors.AccessDeniedError + ) + }) + + it('refuses to import without a source entity path', async function (ctx) { + delete ctx.linkedFileData.source_entity_path + await expect(ctx.createLinkedFile()).to.be.rejectedWith( + ctx.LinkedFilesErrors.BadDataError + ) + }) + }) + + describe('with the source in the project tree', function () { + beforeEach(function (ctx) { + ctx.historySource = false + }) + + it('imports the content of a doc', async function (ctx) { + ctx.ProjectLocator.promises.findElementByPath.resolves({ + element: { _id: 'source-doc-id' }, + type: 'doc', + }) + + expect(await ctx.createLinkedFile()).to.equal('new-file-id') + + expect( + ctx.DocumentUpdaterHandler.promises.flushProjectToMongo + ).to.have.been.calledWith('source-project-id') + expect( + ctx.ProjectLocator.promises.findElementByPath + ).to.have.been.calledWith({ + project_id: 'source-project-id', + path: '/test.txt', + exactCaseMatch: true, + }) + expect( + ctx.LinkedFilesHandler.promises.importContent + ).to.have.been.calledWith( + ctx.projectId, + 'source\ncontent', + ctx.linkedFileData, + ctx.fileName, + ctx.parentFolderId, + ctx.userId + ) + }) + + it('imports the content of a file from its blob', async function (ctx) { + ctx.ProjectLocator.promises.findElementByPath.resolves({ + element: { _id: 'source-file-id', hash: SOURCE_BLOB_HASH }, + type: 'file', + }) + + expect(await ctx.createLinkedFile()).to.equal('new-file-id') + + expect(ctx.HistoryManager.promises.requestBlob).to.have.been.calledWith( + 'source-history-id', + SOURCE_BLOB_HASH + ) + expect(ctx.LinkedFilesHandler.promises.importFromStream).to.have.been + .called + }) + + it('reports a missing source file', async function (ctx) { + ctx.ProjectLocator.promises.findElementByPath.rejects( + new ctx.Errors.NotFoundError() + ) + await expect(ctx.createLinkedFile()).to.be.rejectedWith( + ctx.LinkedFilesErrors.SourceFileNotFoundError + ) + }) + + it('reports a source path that is a folder', async function (ctx) { + ctx.ProjectLocator.promises.findElementByPath.resolves({ + element: { _id: 'source-folder-id' }, + type: 'folder', + }) + await expect(ctx.createLinkedFile()).to.be.rejectedWith( + ctx.LinkedFilesErrors.BadEntityTypeError + ) + }) + + it('does not read from history', async function (ctx) { + ctx.ProjectLocator.promises.findElementByPath.resolves({ + element: { _id: 'source-doc-id' }, + type: 'doc', + }) + await ctx.createLinkedFile() + + expect(ctx.HistoryManager.promises.flushProject).to.not.have.been.called + expect(ctx.HistoryManager.promises.getLatestHistoryWithHistoryId).to.not + .have.been.called + }) + }) + + describe('with the source in the history snapshot', function () { + beforeEach(function (ctx) { + ctx.historySource = true + ctx.HistoryManager.promises.getLatestHistoryWithHistoryId.resolves( + rawChunk({ + 'test.txt': { + hash: SOURCE_BLOB_HASH, + stringLength: SOURCE_CONTENT.length, + }, + }) + ) + }) + + it('flushes the source project into history before reading it', async function (ctx) { + await ctx.createLinkedFile() + + expect( + ctx.DocumentUpdaterHandler.promises.flushProjectToMongo + ).to.have.been.calledWith('source-project-id') + expect(ctx.HistoryManager.promises.flushProject).to.have.been.calledWith( + 'source-project-id' + ) + expect( + ctx.HistoryManager.promises.getLatestHistoryWithHistoryId + ).to.have.been.calledWith('source-history-id') + expect( + ctx.HistoryManager.promises.flushProject + ).to.have.been.calledBefore( + ctx.HistoryManager.promises.getLatestHistoryWithHistoryId + ) + }) + + it('does not look up the source path in the project tree', async function (ctx) { + await ctx.createLinkedFile() + + expect(ctx.ProjectLocator.promises.findElementByPath).to.not.have.been + .called + }) + + it('imports the content of an editable file', async function (ctx) { + expect(await ctx.createLinkedFile()).to.equal('new-file-id') + + expect(ctx.blobStoreHistoryId).to.equal('source-history-id') + expect(ctx.blobStoreHashes).to.deep.equal([SOURCE_BLOB_HASH]) + expect( + ctx.LinkedFilesHandler.promises.importContent + ).to.have.been.calledWith( + ctx.projectId, + SOURCE_CONTENT, + ctx.linkedFileData, + ctx.fileName, + ctx.parentFolderId, + ctx.userId + ) + expect(ctx.HistoryManager.promises.requestBlob).to.not.have.been.called + }) + + it('streams the blob of a file that is not editable', async function (ctx) { + ctx.HistoryManager.promises.getLatestHistoryWithHistoryId.resolves( + rawChunk({ + 'test.txt': { + hash: SOURCE_BLOB_HASH, + byteLength: SOURCE_CONTENT.length, + }, + }) + ) + + expect(await ctx.createLinkedFile()).to.equal('new-file-id') + + expect(ctx.HistoryManager.promises.requestBlob).to.have.been.calledWith( + 'source-history-id', + SOURCE_BLOB_HASH + ) + expect(ctx.LinkedFilesHandler.promises.importFromStream).to.have.been + .called + expect(ctx.LinkedFilesHandler.promises.importContent).to.not.have.been + .called + expect(ctx.blobStoreHashes).to.deep.equal([]) + }) + + it('imports the content of a file with pending edits', async function (ctx) { + ctx.HistoryManager.promises.getLatestHistoryWithHistoryId.resolves( + rawChunk( + { + 'test.txt': { + hash: SOURCE_BLOB_HASH, + stringLength: SOURCE_CONTENT.length, + }, + }, + [ + { + operations: [ + { + pathname: 'test.txt', + textOperation: ['edited ', SOURCE_CONTENT.length], + }, + ], + timestamp: '2026-08-11T09:00:00.000Z', + }, + ] + ) + ) + + expect(await ctx.createLinkedFile()).to.equal('new-file-id') + + expect(ctx.blobStoreHistoryId).to.equal('source-history-id') + expect(ctx.blobStoreHashes).to.deep.equal([SOURCE_BLOB_HASH]) + expect( + ctx.LinkedFilesHandler.promises.importContent + ).to.have.been.calledWith( + ctx.projectId, + `edited ${SOURCE_CONTENT}`, + ctx.linkedFileData, + ctx.fileName, + ctx.parentFolderId, + ctx.userId + ) + expect(ctx.HistoryManager.promises.requestBlob).to.not.have.been.called + }) + + it('accepts a source path without a leading slash', async function (ctx) { + ctx.linkedFileData.source_entity_path = 'test.txt' + expect(await ctx.createLinkedFile()).to.equal('new-file-id') + }) + + it('reports a missing source file', async function (ctx) { + ctx.linkedFileData.source_entity_path = '/other.txt' + await expect(ctx.createLinkedFile()).to.be.rejectedWith( + ctx.LinkedFilesErrors.SourceFileNotFoundError + ) + }) + + it('reports a source path that is a folder', async function (ctx) { + ctx.HistoryManager.promises.getLatestHistoryWithHistoryId.resolves( + rawChunk({ + 'folder/test.txt': { + hash: SOURCE_BLOB_HASH, + stringLength: SOURCE_CONTENT.length, + }, + }) + ) + ctx.linkedFileData.source_entity_path = '/folder' + await expect(ctx.createLinkedFile()).to.be.rejectedWith( + ctx.LinkedFilesErrors.BadEntityTypeError + ) + }) + + it('reads the snapshot when refreshing too', async function (ctx) { + expect( + await ctx.ProjectFileAgent.promises.refreshLinkedFile( + ctx.projectId, + ctx.linkedFileData, + ctx.fileName, + ctx.parentFolderId, + ctx.userId, + true + ) + ).to.equal('new-file-id') + + expect(ctx.HistoryManager.promises.flushProject).to.have.been.calledWith( + 'source-project-id' + ) + }) + }) +}) diff --git a/services/web/types/web-module.ts b/services/web/types/web-module.ts index f6b59cdf6f..33d3182625 100644 --- a/services/web/types/web-module.ts +++ b/services/web/types/web-module.ts @@ -7,6 +7,7 @@ type LinkedFileAgent = { name: string, parentFolderId: string, userId: string, + historySource: boolean, callback: () => void ) => void refreshLinkedFile: ( @@ -15,6 +16,7 @@ type LinkedFileAgent = { name: string, parentFolderId: string, userId: string, + historySource: boolean, callback: () => void ) => void promises: { @@ -23,14 +25,16 @@ type LinkedFileAgent = { linkedFileData: object, name: string, parentFolderId: string, - userId: string + userId: string, + historySource: boolean ) => Promise refreshLinkedFile: ( projectId: string, linkedFileData: object, name: string, parentFolderId: string, - userId: string + userId: string, + historySource: boolean ) => Promise } }