diff --git a/services/web/app/src/Features/History/HistoryRouter.mjs b/services/web/app/src/Features/History/HistoryRouter.mjs new file mode 100644 index 0000000000..3e532a2cad --- /dev/null +++ b/services/web/app/src/Features/History/HistoryRouter.mjs @@ -0,0 +1,166 @@ +// @ts-check + +import Settings from '@overleaf/settings' +import { Joi, validate } from '../../infrastructure/Validation.js' +import { RateLimiter } from '../../infrastructure/RateLimiter.js' +import AuthenticationController from '../Authentication/AuthenticationController.js' +import AuthorizationMiddleware from '../Authorization/AuthorizationMiddleware.js' +import RateLimiterMiddleware from '../Security/RateLimiterMiddleware.js' +import HistoryController from './HistoryController.js' + +const rateLimiters = { + downloadProjectRevision: new RateLimiter('download-project-revision', { + points: 30, + duration: 60 * 60, + }), + getProjectBlob: new RateLimiter('get-project-blob', { + // Download project in full once per hour + points: Settings.maxEntitiesPerProject, + duration: 60 * 60, + }), + flushHistory: new RateLimiter('flush-project-history', { + points: 30, + duration: 60, + }), +} + +function apply(webRouter, privateApiRouter) { + // Blobs + + webRouter.head( + '/project/:project_id/blob/:hash', + validate({ + params: Joi.object({ + project_id: Joi.objectId().required(), + hash: Joi.string().required().hex().length(40), + }), + query: Joi.object({ + fallback: Joi.objectId().optional(), + }), + }), + RateLimiterMiddleware.rateLimit(rateLimiters.getProjectBlob), + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.headBlob + ) + webRouter.get( + '/project/:project_id/blob/:hash', + validate({ + params: Joi.object({ + project_id: Joi.objectId().required(), + hash: Joi.string().required().hex().length(40), + }), + query: Joi.object({ + fallback: Joi.objectId().optional(), + }), + }), + RateLimiterMiddleware.rateLimit(rateLimiters.getProjectBlob), + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.getBlob + ) + + // History diffs + + webRouter.get( + '/project/:Project_id/updates', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.proxyToHistoryApiAndInjectUserDetails + ) + webRouter.get( + '/project/:Project_id/doc/:doc_id/diff', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.proxyToHistoryApi + ) + webRouter.get( + '/project/:Project_id/diff', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.proxyToHistoryApiAndInjectUserDetails + ) + webRouter.get( + '/project/:Project_id/filetree/diff', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.proxyToHistoryApi + ) + + // File and project restore + + webRouter.post( + '/project/:project_id/restore_file', + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + HistoryController.restoreFileFromV2 + ) + webRouter.post( + '/project/:project_id/revert_file', + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + HistoryController.revertFile + ) + webRouter.post( + '/project/:project_id/revert-project', + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + HistoryController.revertProject + ) + + // History download + + webRouter.get( + '/project/:project_id/version/:version/zip', + RateLimiterMiddleware.rateLimit(rateLimiters.downloadProjectRevision), + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.downloadZipOfVersion + ) + + // History flush and resync + + webRouter.post( + '/project/:Project_id/flush', + RateLimiterMiddleware.rateLimit(rateLimiters.flushHistory), + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.proxyToHistoryApi + ) + privateApiRouter.post( + '/project/:Project_id/history/resync', + AuthenticationController.requirePrivateApiAuth(), + HistoryController.resyncProjectHistory + ) + + // History labels + + webRouter.get( + '/project/:Project_id/labels', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.getLabels + ) + webRouter.post( + '/project/:Project_id/labels', + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + HistoryController.createLabel + ) + webRouter.delete( + '/project/:Project_id/labels/:label_id', + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + HistoryController.deleteLabel + ) + + // History snapshot + + webRouter.get( + '/project/:project_id/latest/history', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.proxyToHistoryApi + ) + webRouter.get( + '/project/:project_id/changes', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + HistoryController.proxyToHistoryApi + ) +} + +export default { apply } diff --git a/services/web/app/src/router.mjs b/services/web/app/src/router.mjs index 2ec36f55ee..da3ac2958b 100644 --- a/services/web/app/src/router.mjs +++ b/services/web/app/src/router.mjs @@ -32,6 +32,7 @@ import ProjectDownloadsController from './Features/Downloads/ProjectDownloadsCon import FileStoreController from './Features/FileStore/FileStoreController.mjs' import DocumentUpdaterController from './Features/DocumentUpdater/DocumentUpdaterController.mjs' import HistoryController from './Features/History/HistoryController.js' +import HistoryRouter from './Features/History/HistoryRouter.mjs' import ExportsController from './Features/Exports/ExportsController.mjs' import PasswordResetRouter from './Features/PasswordReset/PasswordResetRouter.mjs' import StaticPagesRouter from './Features/StaticPages/StaticPagesRouter.mjs' @@ -120,24 +121,6 @@ const rateLimiters = { points: 10, duration: 60, }), - downloadProjectRevision: new RateLimiter('download-project-revision', { - points: 30, - duration: 60 * 60, - }), - flushHistory: new RateLimiter('flush-project-history', { - // Allow flushing once every 30s-1s (allow for network jitter). - points: 1, - duration: 30 - 1, - }), - getProjectBlob: new RateLimiter('get-project-blob', { - // Download project in full once per hour - points: Settings.maxEntitiesPerProject, - duration: 60 * 60, - }), - getHistorySnapshot: new RateLimiter( - 'get-history-snapshot', - openProjectRateLimiter.getOptions() - ), endorseEmail: new RateLimiter('endorse-email', { points: 30, duration: 60, @@ -552,36 +535,10 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) { HistoryController.fileToBlobRedirectMiddleware, FileStoreController.getFile ) - webRouter.head( - '/project/:project_id/blob/:hash', - validate({ - params: Joi.object({ - project_id: Joi.objectId().required(), - hash: Joi.string().required().hex().length(40), - }), - query: Joi.object({ - fallback: Joi.objectId().optional(), - }), - }), - RateLimiterMiddleware.rateLimit(rateLimiters.getProjectBlob), - AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.headBlob - ) - webRouter.get( - '/project/:project_id/blob/:hash', - validate({ - params: Joi.object({ - project_id: Joi.objectId().required(), - hash: Joi.string().required().hex().length(40), - }), - query: Joi.object({ - fallback: Joi.objectId().optional(), - }), - }), - RateLimiterMiddleware.rateLimit(rateLimiters.getProjectBlob), - AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.getBlob - ) + + // Has to be applied after any route using fileToBlobRedirectMiddleware + HistoryRouter.apply(webRouter, privateApiRouter) + webRouter.get( '/Project/:Project_id/doc/:Doc_id/download', // "download" suffix to avoid conflict with private API route at doc/:doc_id AuthorizationMiddleware.ensureUserCanReadProject, @@ -801,75 +758,6 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) { AuthorizationMiddleware.ensureUserCanAdminProject, ProjectController.renameProject ) - webRouter.get( - '/project/:Project_id/updates', - AuthorizationMiddleware.blockRestrictedUserFromProject, - AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.proxyToHistoryApiAndInjectUserDetails - ) - webRouter.get( - '/project/:Project_id/doc/:doc_id/diff', - AuthorizationMiddleware.blockRestrictedUserFromProject, - AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.proxyToHistoryApi - ) - webRouter.get( - '/project/:Project_id/diff', - AuthorizationMiddleware.blockRestrictedUserFromProject, - AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.proxyToHistoryApiAndInjectUserDetails - ) - webRouter.get( - '/project/:Project_id/filetree/diff', - AuthorizationMiddleware.blockRestrictedUserFromProject, - AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.proxyToHistoryApi - ) - webRouter.post( - '/project/:project_id/restore_file', - AuthorizationMiddleware.ensureUserCanWriteProjectContent, - HistoryController.restoreFileFromV2 - ) - webRouter.post( - '/project/:project_id/revert_file', - AuthorizationMiddleware.ensureUserCanWriteProjectContent, - HistoryController.revertFile - ) - webRouter.post( - '/project/:project_id/revert-project', - AuthorizationMiddleware.ensureUserCanWriteProjectContent, - HistoryController.revertProject - ) - webRouter.get( - '/project/:project_id/version/:version/zip', - RateLimiterMiddleware.rateLimit(rateLimiters.downloadProjectRevision), - AuthorizationMiddleware.blockRestrictedUserFromProject, - AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.downloadZipOfVersion - ) - privateApiRouter.post( - '/project/:Project_id/history/resync', - AuthenticationController.requirePrivateApiAuth(), - HistoryController.resyncProjectHistory - ) - - webRouter.get( - '/project/:Project_id/labels', - AuthorizationMiddleware.blockRestrictedUserFromProject, - AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.getLabels - ) - webRouter.post( - '/project/:Project_id/labels', - AuthorizationMiddleware.ensureUserCanWriteProjectContent, - HistoryController.createLabel - ) - webRouter.delete( - '/project/:Project_id/labels/:label_id', - AuthorizationMiddleware.ensureUserCanWriteProjectContent, - HistoryController.deleteLabel - ) - webRouter.post( '/project/:project_id/export/:brand_variation_id', AuthorizationMiddleware.ensureUserCanWriteProjectContent, diff --git a/services/web/frontend/js/infrastructure/project-snapshot.ts b/services/web/frontend/js/infrastructure/project-snapshot.ts new file mode 100644 index 0000000000..226d44923c --- /dev/null +++ b/services/web/frontend/js/infrastructure/project-snapshot.ts @@ -0,0 +1,124 @@ +import pLimit from 'p-limit' +import { Change, Chunk, Snapshot } from 'overleaf-editor-core' +import { RawChange, RawChunk } from 'overleaf-editor-core/lib/types' +import { FetchError, getJSON, postJSON } from '@/infrastructure/fetch-json' + +const DOWNLOAD_BLOBS_CONCURRENCY = 10 + +/** + * Project snapshot container with on-demand refresh + */ +export class ProjectSnapshot { + private projectId: string + private snapshot: Snapshot + private version: number + private state: 'init' | 'refreshing' | 'ready' + private blobStore: SimpleBlobStore + + constructor(projectId: string) { + this.projectId = projectId + this.snapshot = new Snapshot() + this.version = 0 + this.state = 'init' + this.blobStore = new SimpleBlobStore(this.projectId) + } + + async refresh() { + if (this.state === 'refreshing') { + // Prevent concurrent refreshes + return + } + + await flushHistory(this.projectId) + + if (this.state === 'init') { + const chunk = await fetchLatestChunk(this.projectId) + this.snapshot = chunk.getSnapshot() + this.snapshot.applyAll(chunk.getChanges()) + this.version = chunk.getEndVersion() + } else { + const changes = await fetchLatestChanges(this.projectId, this.version) + this.snapshot.applyAll(changes) + this.version += changes.length + } + + this.state = 'ready' + await this.loadDocs() + } + + getDocPaths(): string[] { + const allPaths = this.snapshot.getFilePathnames() + return allPaths.filter(path => this.snapshot.getFile(path)?.isEditable()) + } + + getDocContents(path: string): string | null { + const file = this.snapshot.getFile(path) + if (file == null) { + return null + } + return file.getContent() ?? null + } + + private async loadDocs() { + const paths = this.getDocPaths() + const limit = pLimit(DOWNLOAD_BLOBS_CONCURRENCY) + await Promise.all( + paths.map(path => + limit(async () => { + const file = this.snapshot.getFile(path) + await file?.load('eager', this.blobStore) + }) + ) + ) + } +} + +/** + * Blob store that fetches blobs from the history service + */ +class SimpleBlobStore { + private projectId: string + + constructor(projectId: string) { + this.projectId = projectId + } + + async getString(hash: string): Promise { + return await fetchBlob(this.projectId, hash) + } + + async getObject(hash: string) { + const blob = await this.getString(hash) + return JSON.parse(blob) + } +} + +async function flushHistory(projectId: string) { + await postJSON(`/project/${projectId}/flush`) +} + +async function fetchLatestChunk(projectId: string): Promise { + const response = await getJSON<{ chunk: RawChunk }>( + `/project/${projectId}/latest/history` + ) + return Chunk.fromRaw(response.chunk) +} + +async function fetchLatestChanges( + projectId: string, + version: number +): Promise { + const response = await getJSON( + `/project/${projectId}/changes?since=${version}` + ) + return response.map(Change.fromRaw).filter(change => change != null) +} + +async function fetchBlob(projectId: string, hash: string): Promise { + const url = `/project/${projectId}/blob/${hash}` + const res = await fetch(url) + if (!res.ok) { + throw new FetchError('Failed to fetch blob', url, undefined, res) + } + return await res.text() +} diff --git a/services/web/test/frontend/infrastructure/project-snapshot.test.ts b/services/web/test/frontend/infrastructure/project-snapshot.test.ts new file mode 100644 index 0000000000..5038738ed2 --- /dev/null +++ b/services/web/test/frontend/infrastructure/project-snapshot.test.ts @@ -0,0 +1,187 @@ +import { expect } from 'chai' +import fetchMock from 'fetch-mock' +import { ProjectSnapshot } from '@/infrastructure/project-snapshot' + +describe('ProjectSnapshot', function () { + let snapshot: ProjectSnapshot + const projectId = 'project-id' + + beforeEach(function () { + snapshot = new ProjectSnapshot(projectId) + }) + + describe('before initialization', function () { + describe('getDocPaths()', function () { + it('returns an empty string', function () { + expect(snapshot.getDocPaths()).to.deep.equal([]) + }) + }) + + describe('getDocContents()', function () { + it('returns null', function () { + expect(snapshot.getDocContents('main.tex')).to.be.null + }) + }) + }) + + const files = { + 'main.tex': { + contents: '\\documentclass{article}\netc.', + hash: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + }, + 'hello.txt': { + contents: 'Hello history!', + hash: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + }, + 'goodbye.txt': { + contents: "We're done here", + hash: 'dddddddddddddddddddddddddddddddddddddddd', + }, + } + + const chunk = { + history: { + snapshot: { + files: {}, + }, + changes: [ + { + operations: [ + { + pathname: 'hello.txt', + file: { + hash: files['hello.txt'].hash, + stringLength: files['hello.txt'].contents.length, + }, + }, + { + pathname: 'main.tex', + file: { + hash: files['main.tex'].hash, + stringLength: files['main.tex'].contents.length, + }, + }, + { + pathname: 'frog.jpg', + file: { + hash: 'cccccccccccccccccccccccccccccccccccccccc', + byteLength: 97080, + }, + }, + ], + timestamp: '2025-01-01T12:00:00.000Z', + }, + ], + }, + startVersion: 0, + } + + async function initializeSnapshot() { + fetchMock.postOnce(`/project/${projectId}/flush`, 200) + fetchMock.getOnce(`/project/${projectId}/latest/history`, { chunk }) + fetchMock.getOnce( + `/project/${projectId}/blob/${files['main.tex'].hash}`, + files['main.tex'].contents + ) + fetchMock.getOnce( + `/project/${projectId}/blob/${files['hello.txt'].hash}`, + files['hello.txt'].contents + ) + await snapshot.refresh() + expect(fetchMock.done()).to.be.true + fetchMock.reset() + } + + describe('after initialization', function () { + beforeEach(initializeSnapshot) + + describe('getDocPaths()', function () { + it('returns the editable docs', function () { + expect(snapshot.getDocPaths()).to.have.members([ + 'main.tex', + 'hello.txt', + ]) + }) + }) + + describe('getDocContents()', function () { + it('returns the doc contents', function () { + expect(snapshot.getDocContents('main.tex')).to.equal( + files['main.tex'].contents + ) + }) + + it('returns null for binary files', function () { + expect(snapshot.getDocContents('frog.jpg')).to.be.null + }) + + it('returns null for inexistent files', function () { + expect(snapshot.getDocContents('does-not-exist.txt')).to.be.null + }) + }) + }) + + const changes = [ + { + operations: [ + { + pathname: 'hello.txt', + textOperation: ['Quote: ', files['hello.txt'].contents.length], + }, + { + pathname: 'goodbye.txt', + file: { + hash: files['goodbye.txt'].hash, + stringLength: files['goodbye.txt'].contents.length, + }, + }, + ], + timestamp: '2025-01-01T13:00:00.000Z', + }, + ] + + async function refreshSnapshot() { + fetchMock.postOnce(`/project/${projectId}/flush`, 200, { repeat: 2 }) + fetchMock.getOnce(`/project/${projectId}/changes?since=1`, changes) + fetchMock.getOnce( + `/project/${projectId}/blob/${files['goodbye.txt'].hash}`, + files['goodbye.txt'].contents + ) + await snapshot.refresh() + expect(fetchMock.done()).to.be.true + fetchMock.reset() + } + + describe('after refresh', function () { + beforeEach(initializeSnapshot) + beforeEach(refreshSnapshot) + + afterEach(function () { + fetchMock.reset() + }) + + describe('getDocPaths()', function () { + it('returns the editable docs', function () { + expect(snapshot.getDocPaths()).to.have.members([ + 'main.tex', + 'hello.txt', + 'goodbye.txt', + ]) + }) + }) + + describe('getDocCotents()', function () { + it('returns the up to date content', function () { + expect(snapshot.getDocContents('hello.txt')).to.equal( + `Quote: ${files['hello.txt'].contents}` + ) + }) + + it('returns contents of new files', function () { + expect(snapshot.getDocContents('goodbye.txt')).to.equal( + files['goodbye.txt'].contents + ) + }) + }) + }) +})