agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-14 14:13:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use history snapshot when doing file/project restore (#28502)

Browse Source 
* Add getDocUpdaterCompatibleRanges utility function

* use history snapshot for file/project restore

* move overleaf-editor-core from devDependencies

GitOrigin-RevId: 62481a5304ada9d931e018418be3c0719bccf1f3
This commit is contained in:
Domagoj Kriskovic
2025-09-19 08:07:58 +00:00
committed by Copybot
parent a6e9a5c7e9
commit 484a01a173
7 changed files with 707 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/project-history/app/js/SnapshotManager.js
+3 -119
View File
@@ -81,126 +81,10 @@ async function getRangesSnapshot(projectId, version, pathname) {
}
const historyId = await WebApiManager.promises.getHistoryId(projectId)
await file.load('eager', HistoryStoreManager.getBlobStore(historyId))
const content = file.getContent()
if (content == null) {
throw new Error('Unable to read file contents')
}
const trackedChanges = file.getTrackedChanges().asSorted()
const comments = file.getComments().toArray()
const docUpdaterCompatibleTrackedChanges = []
let trackedDeletionOffset = 0
for (const trackedChange of trackedChanges) {
const isTrackedDeletion = trackedChange.tracking.type === 'delete'
const trackedChangeContent = content.slice(
trackedChange.range.start,
trackedChange.range.end
)
const tcContent = isTrackedDeletion
? { d: trackedChangeContent }
: { i: trackedChangeContent }
docUpdaterCompatibleTrackedChanges.push({
op: {
p: trackedChange.range.start - trackedDeletionOffset,
...tcContent,
},
metadata: {
ts: trackedChange.tracking.ts.toISOString(),
user_id: trackedChange.tracking.userId,
},
})
if (isTrackedDeletion) {
trackedDeletionOffset += trackedChange.range.length
}
}
// Comments are shifted left by the length of any previous tracked deletions.
// If they overlap with a tracked deletion, they are truncated.
//
// Example:
// { } comment
// [ ] tracked deletion
// the quic[k {b]rown [fox] jum[ps} ove]r the lazy dog
// => rown jum
// starting at position 8
const trackedDeletions = trackedChanges.filter(
tc => tc.tracking.type === 'delete'
)
const docUpdaterCompatibleComments = []
for (const comment of comments) {
let trackedDeletionIndex = 0
if (comment.ranges.length === 0) {
// Translate detached comments into zero length comments at position 0
docUpdaterCompatibleComments.push({
op: {
p: 0,
c: '',
t: comment.id,
resolved: comment.resolved,
},
})
continue
}
// Consider a multiple range comment as a single comment that joins all its
// ranges
const commentStart = comment.ranges[0].start
const commentEnd = comment.ranges[comment.ranges.length - 1].end
let commentContent = ''
// Docupdater position
let position = commentStart
while (trackedDeletions[trackedDeletionIndex]?.range.end <= commentStart) {
// Skip over tracked deletions that are before the current comment range
position -= trackedDeletions[trackedDeletionIndex].range.length
trackedDeletionIndex++
}
if (trackedDeletions[trackedDeletionIndex]?.range.start < commentStart) {
// There's overlap with a tracked deletion, move the position left and
// truncate the overlap
position -=
commentStart - trackedDeletions[trackedDeletionIndex].range.start
}
// Cursor in the history content
let cursor = commentStart
while (cursor < commentEnd) {
const trackedDeletion = trackedDeletions[trackedDeletionIndex]
if (!trackedDeletion || trackedDeletion.range.start >= commentEnd) {
// We've run out of relevant tracked changes
commentContent += content.slice(cursor, commentEnd)
break
}
if (trackedDeletion.range.start > cursor) {
// There's a gap between the current cursor and the tracked deletion
commentContent += content.slice(cursor, trackedDeletion.range.start)
}
if (trackedDeletion.range.end <= commentEnd) {
// Skip to the end of the tracked delete
cursor = trackedDeletion.range.end
trackedDeletionIndex++
} else {
// We're done with that comment
break
}
}
docUpdaterCompatibleComments.push({
op: {
p: position,
c: commentContent,
t: comment.id,
resolved: comment.resolved,
},
id: comment.id,
})
}
return {
changes: docUpdaterCompatibleTrackedChanges,
comments: docUpdaterCompatibleComments,
}
// Use the utility function from overleaf-editor-core
const { changes, comments } = Core.getDocUpdaterCompatibleRanges(file)
return { changes, comments }
}
/**
services/web/app/src/Features/History/RestoreManager.mjs
+37 -27
View File
@@ -18,6 +18,8 @@ import ChatManager from '../Chat/ChatManager.js'
import OError from '@overleaf/o-error'
import ProjectGetter from '../Project/ProjectGetter.js'
import ProjectEntityHandler from '../Project/ProjectEntityHandler.js'
import HistoryManager from './HistoryManager.js'
import { Snapshot, getDocUpdaterCompatibleRanges } from 'overleaf-editor-core'
async function getCommentThreadIds(projectId) {
await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId)
@@ -60,22 +62,41 @@ const RestoreManager = {
async revertFile(userId, projectId, version, pathname, options = {}) {
const threadIds = await getCommentThreadIds(projectId)
const snapshotRaw = await HistoryManager.promises.getContentAtVersion(
projectId,
version
)
const snapshot = Snapshot.fromRaw(snapshotRaw)
return await RestoreManager._revertSingleFile(
userId,
projectId,
version,
pathname,
threadIds,
snapshot,
options
)
},
/**
*
* @param {string} userId
* @param {string} projectId
* @param {string} version
* @param {string} pathname
* @param {Set<string>} threadIds
* @param {Snapshot} projectSnapshotAtVersion
* @param {object} options
*/
async _revertSingleFile(
userId,
projectId,
version,
pathname,
threadIds,
projectSnapshotAtVersion,
options = {}
) {
const endTimer = Metrics.revertFileDurationSeconds.startTimer()
@@ -156,16 +177,16 @@ const RestoreManager = {
threadIds.delete(file.element._id.toString())
}
const { metadata } = await RestoreManager._getMetadataFromHistory(
projectId,
version,
pathname
)
const snapshotFile = projectSnapshotAtVersion.getFile(pathname)
if (!snapshotFile) {
throw new OError('file not found in snapshot', { pathname })
}
// Look for metadata indicating a linked file.
const isFileMetadata = metadata && 'provider' in metadata
const fileMetadata = snapshotFile.getMetadata()
const isFileMetadata = fileMetadata && 'provider' in fileMetadata
logger.debug({ metadata }, 'metadata from history')
logger.debug({ fileMetadata }, 'metadata from history')
if (importInfo.type === 'file' || isFileMetadata) {
const newFile = await EditorController.promises.upsertFile(
@@ -173,7 +194,7 @@ const RestoreManager = {
parentFolderId,
basename,
fsPath,
metadata,
fileMetadata,
origin,
userId
)
@@ -185,11 +206,7 @@ const RestoreManager = {
}
}
const ranges = await RestoreManager._getRangesFromHistory(
projectId,
version,
pathname
)
const ranges = getDocUpdaterCompatibleRanges(snapshotFile)
const documentCommentIds = new Set(
ranges.comments?.map(({ op: { t } }) => t)
@@ -364,6 +381,12 @@ const RestoreManager = {
}
const threadIds = await getCommentThreadIds(projectId)
const snapshotRaw = await HistoryManager.promises.getContentAtVersion(
projectId,
version
)
const snapshot = Snapshot.fromRaw(snapshotRaw)
const reverted = []
for (const pathname of pathsAtPastVersion) {
const res = await RestoreManager._revertSingleFile(
@@ -372,6 +395,7 @@ const RestoreManager = {
version,
pathname,
threadIds,
snapshot,
{ origin }
)
reverted.push({
@@ -414,20 +438,6 @@ const RestoreManager = {
return await FileWriter.promises.writeUrlToDisk(projectId, url)
},
async _getRangesFromHistory(projectId, version, pathname) {
const url = `${
Settings.apis.project_history.url
}/project/${projectId}/ranges/version/${version}/${encodeURIComponent(pathname)}`
return await fetchJson(url)
},
async _getMetadataFromHistory(projectId, version, pathname) {
const url = `${
Settings.apis.project_history.url
}/project/${projectId}/metadata/version/${version}/${encodeURIComponent(pathname)}`
return await fetchJson(url)
},
async _getUpdatesFromHistory(projectId, version) {
const url = `${Settings.apis.project_history.url}/project/${projectId}/updates?before=${version}&min_count=1`
const res = await fetchJson(url)
services/web/package.json
+1 -1
View File
@@ -158,6 +158,7 @@
"nodemailer": "^6.7.0",
"on-headers": "^1.0.2",
"otplib": "^12.0.1",
"overleaf-editor-core": "*",
"p-limit": "^2.3.0",
"p-props": "4.0.0",
"p-queue": "^8.1.0",
@@ -344,7 +345,6 @@
"mock-fs": "^5.1.2",
"nock": "^13.5.6",
"nvd3": "^1.8.6",
"overleaf-editor-core": "*",
"p-reflect": "^3.1.0",
"pdfjs-dist": "5.1.91",
"pirates": "^4.0.1",
services/web/test/unit/src/History/RestoreManager.test.mjs
+115 -59
View File
@@ -20,6 +20,60 @@ describe('RestoreManager', function () {
default: Errors,
}))
vi.doMock('../../../../app/src/Features/History/HistoryManager.js', () => ({
default: (ctx.HistoryManager = {
promises: {
getContentAtVersion: sinon.stub().resolves({
// Raw snapshot data that will be passed to Snapshot.fromRaw
files: {
'main.tex': {
hash: 'abcdef1234567890abcdef1234567890abcdef12',
stringLength: 100,
metadata: {
editorId: 'test-editor',
},
},
'foo.tex': {
hash: 'abcdef1234567890abcdef1234567890abcdef12',
stringLength: 100,
metadata: {
editorId: 'test-editor',
},
},
'folder/file.tex': {
hash: 'abcdef1234567890abcdef1234567890abcdef12',
stringLength: 100,
metadata: {
editorId: 'test-editor',
},
},
'foo.png': {
hash: 'abcdef1234567890abcdef1234567890abcdef12',
stringLength: 100,
metadata: {
provider: 'bar',
},
},
'linkedFile.bib': {
hash: 'abcdef1234567890abcdef1234567890abcdef12',
stringLength: 100,
metadata: {
provider: 'mendeley',
},
},
'withMainTrue.tex': {
hash: 'abcdef1234567890abcdef1234567890abcdef12',
stringLength: 100,
metadata: {
main: true,
},
},
},
}),
},
}),
}))
vi.doMock('../../../../app/src/infrastructure/Metrics.js', () => ({
default: {
revertFileDurationSeconds: {
@@ -101,10 +155,46 @@ describe('RestoreManager', function () {
})
)
vi.doMock('overleaf-editor-core', () => ({
Snapshot: {
fromRaw: sinon.stub().callsFake(snapshotData => ({
getFile: pathname => ({
getStringLength: sinon.stub().returns(100),
getByteLength: sinon.stub().returns(100),
getContent: sinon.stub().returns('file content'),
isEditable: sinon.stub().returns(true),
getMetadata: sinon
.stub()
.returns(snapshotData?.files?.[pathname]?.metadata),
}),
})),
},
getDocUpdaterCompatibleRanges: (ctx.getDocUpdaterCompatibleRanges = sinon
.stub()
.returns({
changes: ctx.tracked_changes || [],
comments: ctx.comments || [],
})),
}))
ctx.RestoreManager = (await import(modulePath)).default
ctx.user_id = 'mock-user-id'
ctx.project_id = 'mock-project-id'
ctx.version = 42
// Add missing method mocks to RestoreManager
ctx.RestoreManager.promises._getUpdatesFromHistory = sinon.stub().resolves([
{
toV: ctx.version,
meta: { end_ts: new Date('2024-01-01T00:00:00.000Z') },
},
])
ctx.RestoreManager.promises._getProjectPathsAtVersion = sinon
.stub()
.resolves([])
ctx.RestoreManager.promises._writeFileVersionToDisk = sinon
.stub()
.resolves('/tmp/mock-file-path')
})
afterEach(function () {
@@ -290,10 +380,6 @@ describe('RestoreManager', function () {
ctx.FileSystemImportManager.promises.addEntity = sinon
.stub()
.resolves((ctx.entity = 'mock-entity'))
ctx.RestoreManager.promises._getRangesFromHistory = sinon.stub().rejects()
ctx.RestoreManager.promises._getMetadataFromHistory = sinon
.stub()
.resolves({ metadata: undefined })
})
describe('reverting a project without ranges support', function () {
@@ -395,12 +481,10 @@ describe('RestoreManager', function () {
ctx.FileSystemImportManager.promises.importFile = sinon
.stub()
.resolves({ type: 'doc', lines: ['foo', 'bar', 'baz'] })
ctx.RestoreManager.promises._getRangesFromHistory = sinon
.stub()
.resolves({
changes: ctx.tracked_changes,
comments: ctx.comments,
})
ctx.getDocUpdaterCompatibleRanges.returns({
changes: ctx.tracked_changes,
comments: ctx.comments,
})
ctx.RestoreManager.promises._getUpdatesFromHistory = sinon
.stub()
.resolves([
@@ -444,12 +528,6 @@ describe('RestoreManager', function () {
it('should return the created entity', function (ctx) {
expect(ctx.data).to.deep.equal(ctx.addedFile)
})
it('should look up ranges', function (ctx) {
expect(
ctx.RestoreManager.promises._getRangesFromHistory
).to.have.been.calledWith(ctx.project_id, ctx.version, ctx.pathname)
})
})
describe('with an existing file in the current project', function () {
@@ -560,12 +638,6 @@ describe('RestoreManager', function () {
it('should return the created entity', function (ctx) {
expect(ctx.data).to.deep.equal(ctx.addedFile)
})
it('should look up ranges', function (ctx) {
expect(
ctx.RestoreManager.promises._getRangesFromHistory
).to.have.been.calledWith(ctx.project_id, ctx.version, ctx.pathname)
})
})
describe('with comments in same doc', function () {
@@ -623,7 +695,14 @@ describe('RestoreManager', function () {
ctx.project_id,
ctx.version,
ctx.pathname,
ctx.threadIds
ctx.threadIds,
{
getFile: sinon.stub().returns({
getMetadata: sinon.stub().returns(undefined),
getContent: sinon.stub().returns('file content'),
isEditable: sinon.stub().returns(true),
}),
}
)
})
@@ -687,7 +766,14 @@ describe('RestoreManager', function () {
ctx.project_id,
ctx.version,
ctx.pathname,
ctx.threadIds
ctx.threadIds,
{
getFile: sinon.stub().returns({
getMetadata: sinon.stub().returns(undefined),
getContent: sinon.stub().returns('file content'),
isEditable: sinon.stub().returns(true),
}),
}
)
})
@@ -803,12 +889,6 @@ describe('RestoreManager', function () {
ctx.EditorController.promises.upsertFile = sinon
.stub()
.resolves({ _id: 'mock-file-id', type: 'file' })
ctx.RestoreManager.promises._getRangesFromHistory = sinon
.stub()
.resolves({
changes: [],
comments: [],
})
ctx.EditorController.promises.addDocWithRanges = sinon
.stub()
.resolves((ctx.addedFile = { _id: 'mock-doc-id', type: 'doc' }))
@@ -831,9 +911,6 @@ describe('RestoreManager', function () {
ctx.FileSystemImportManager.promises.importFile = sinon
.stub()
.resolves({ type: 'file' })
ctx.RestoreManager.promises._getMetadataFromHistory = sinon
.stub()
.resolves({ metadata: { provider: 'bar' } })
ctx.result = await ctx.RestoreManager.promises.revertFile(
ctx.user_id,
ctx.project_id,
@@ -868,11 +945,6 @@ describe('RestoreManager', function () {
)
})
it('should not look up ranges', function (ctx) {
expect(ctx.RestoreManager.promises._getRangesFromHistory).to.not.have
.been.called
})
it('should not try to add a document', function (ctx) {
expect(ctx.EditorController.promises.addDocWithRanges).to.not.have
.been.called
@@ -881,13 +953,10 @@ describe('RestoreManager', function () {
describe('when reverting a linked document with provider', function () {
beforeEach(async function (ctx) {
ctx.pathname = 'foo.tex'
ctx.pathname = 'linkedFile.bib'
ctx.FileSystemImportManager.promises.importFile = sinon
.stub()
.resolves({ type: 'doc', lines: ['foo', 'bar', 'baz'] })
ctx.RestoreManager.promises._getMetadataFromHistory = sinon
.stub()
.resolves({ metadata: { provider: 'bar' } })
ctx.result = await ctx.RestoreManager.promises.revertFile(
ctx.user_id,
ctx.project_id,
@@ -909,9 +978,9 @@ describe('RestoreManager', function () {
).to.have.been.calledWith(
ctx.project_id,
'mock-folder-id',
'foo.tex',
'linkedFile.bib',
ctx.fsPath,
{ provider: 'bar' },
{ provider: 'mendeley' },
{
kind: 'file-restore',
path: ctx.pathname,
@@ -922,11 +991,6 @@ describe('RestoreManager', function () {
)
})
it('should not look up ranges', function (ctx) {
expect(ctx.RestoreManager.promises._getRangesFromHistory).to.not.have
.been.called
})
it('should not try to add a document', function (ctx) {
expect(ctx.EditorController.promises.addDocWithRanges).to.not.have
.been.called
@@ -935,13 +999,10 @@ describe('RestoreManager', function () {
describe('when reverting a linked document with { main: true }', function () {
beforeEach(async function (ctx) {
ctx.pathname = 'foo.tex'
ctx.pathname = 'withMainTrue.tex'
ctx.FileSystemImportManager.promises.importFile = sinon
.stub()
.resolves({ type: 'doc', lines: ['foo', 'bar', 'baz'] })
ctx.RestoreManager.promises._getMetadataFromHistory = sinon
.stub()
.resolves({ metadata: { main: true } })
ctx.result = await ctx.RestoreManager.promises.revertFile(
ctx.user_id,
ctx.project_id,
@@ -962,18 +1023,13 @@ describe('RestoreManager', function () {
.called
})
it('should look up ranges', function (ctx) {
expect(ctx.RestoreManager.promises._getRangesFromHistory).to.have.been
.called
})
it('should add the document', function (ctx) {
expect(
ctx.EditorController.promises.addDocWithRanges
).to.have.been.calledWith(
ctx.project_id,
ctx.folder_id,
'foo.tex',
'withMainTrue.tex',
['foo', 'bar', 'baz'],
{ changes: [], comments: [] }
)