agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-14 22:23:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

[project-history] add support for resync of history-ot ranges (#26475)

Browse Source 
* [project-history] add support for resync of history-ot ranges

* [project-history] avoid compressing sharejs and history-ot upgrades

* [document-updater] improve error message of some assertions

... by migrating the assertions like this:
```diff
-stub.calledWith().should.equal(true)
+stub.should.have.been.calledWith()
```
```diff
-stub.called.should.equal(false)
+stub.should.not.have.been.called
```

* [document-updater] move content field in resyncDocContent

* [document-updater] add support for resync of history-ot ranges

GitOrigin-RevId: e6104686a26934a5f25a8f095cbe00c163fbbaa7
This commit is contained in:
Jakob Ackermann
2025-06-19 08:05:18 +00:00
committed by Copybot
parent f5acd16b47
commit dc9e7bd12d
9 changed files with 1579 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/document-updater/app/js/DocumentManager.js
-5
View File
@@ -535,11 +535,6 @@ const DocumentManager = {
if (opts.historyRangesMigration) {
historyRangesSupport = opts.historyRangesMigration === 'forwards'
}
if (!Array.isArray(lines)) {
const file = StringFileData.fromRaw(lines)
// TODO(24596): tc support for history-ot
lines = file.getLines()
}
await ProjectHistoryRedisManager.promises.queueResyncDocContent(
projectId,
services/document-updater/app/js/Limits.js
+15
View File
@@ -28,4 +28,19 @@ module.exports = {
// since we didn't hit the limit in the loop, the document is within the allowed length
return false
},
/**
* @param {StringFileRawData} raw
* @param {number} maxDocLength
*/
stringFileDataContentIsTooLarge(raw, maxDocLength) {
let n = raw.content.length
if (n <= maxDocLength) return false // definitely under the limit, no need to calculate the total size
for (const tc of raw.trackedChanges ?? []) {
if (tc.tracking.type !== 'delete') continue
n -= tc.range.length
if (n <= maxDocLength) return false // under the limit now, no need to calculate the exact size
}
return true
},
}
services/document-updater/app/js/ProjectHistoryRedisManager.js
+30 -13
View File
@@ -8,13 +8,14 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
)
const logger = require('@overleaf/logger')
const metrics = require('./Metrics')
const { docIsTooLarge } = require('./Limits')
const { docIsTooLarge, stringFileDataContentIsTooLarge } = require('./Limits')
const { addTrackedDeletesToContent, extractOriginOrSource } = require('./Utils')
const HistoryConversions = require('./HistoryConversions')
const OError = require('@overleaf/o-error')
/**
* @import { Ranges } from './types'
* @import { StringFileRawData } from 'overleaf-editor-core/lib/types'
*/
const ProjectHistoryRedisManager = {
@@ -180,7 +181,7 @@ const ProjectHistoryRedisManager = {
* @param {string} projectId
* @param {string} projectHistoryId
* @param {string} docId
* @param {string[]} lines
* @param {string[] | StringFileRawData} lines
* @param {Ranges} ranges
* @param {string[]} resolvedCommentIds
* @param {number} version
@@ -204,13 +205,8 @@ const ProjectHistoryRedisManager = {
'queue doc content resync'
)
let content = lines.join('\n')
if (historyRangesSupport) {
content = addTrackedDeletesToContent(content, ranges.changes ?? [])
}
const projectUpdate = {
resyncDocContent: { content, version },
resyncDocContent: { version },
projectHistoryId,
path: pathname,
doc: docId,
@@ -219,17 +215,38 @@ const ProjectHistoryRedisManager = {
},
}
if (historyRangesSupport) {
projectUpdate.resyncDocContent.ranges =
HistoryConversions.toHistoryRanges(ranges)
projectUpdate.resyncDocContent.resolvedCommentIds = resolvedCommentIds
let content = ''
if (Array.isArray(lines)) {
content = lines.join('\n')
if (historyRangesSupport) {
content = addTrackedDeletesToContent(content, ranges.changes ?? [])
projectUpdate.resyncDocContent.ranges =
HistoryConversions.toHistoryRanges(ranges)
projectUpdate.resyncDocContent.resolvedCommentIds = resolvedCommentIds
}
} else {
content = lines.content
projectUpdate.resyncDocContent.historyOTRanges = {
comments: lines.comments,
trackedChanges: lines.trackedChanges,
}
}
projectUpdate.resyncDocContent.content = content
const jsonUpdate = JSON.stringify(projectUpdate)
// Do an optimised size check on the docLines using the serialised
// project update length as an upper bound
const sizeBound = jsonUpdate.length
if (docIsTooLarge(sizeBound, lines, Settings.max_doc_length)) {
if (Array.isArray(lines)) {
if (docIsTooLarge(sizeBound, lines, Settings.max_doc_length)) {
throw new OError(
'blocking resync doc content insert into project history queue: doc is too large',
{ projectId, docId, docSize: sizeBound }
)
}
} else if (
stringFileDataContentIsTooLarge(lines, Settings.max_doc_length)
) {
throw new OError(
'blocking resync doc content insert into project history queue: doc is too large',
{ projectId, docId, docSize: sizeBound }
services/document-updater/test/unit/js/Limits/LimitsTests.js
+84
View File
@@ -81,4 +81,88 @@ describe('Limits', function () {
})
})
})
describe('stringFileDataContentIsTooLarge', function () {
it('should handle small docs', function () {
expect(
this.Limits.stringFileDataContentIsTooLarge({ content: '' }, 123)
).to.equal(false)
})
it('should handle docs at the limit', function () {
expect(
this.Limits.stringFileDataContentIsTooLarge(
{ content: 'x'.repeat(123) },
123
)
).to.equal(false)
})
it('should handle docs above the limit', function () {
expect(
this.Limits.stringFileDataContentIsTooLarge(
{ content: 'x'.repeat(123 + 1) },
123
)
).to.equal(true)
})
it('should handle docs above the limit and below with tracked-deletes removed', function () {
expect(
this.Limits.stringFileDataContentIsTooLarge(
{
content: 'x'.repeat(123 + 1),
trackedChanges: [
{
range: { pos: 1, length: 1 },
tracking: {
type: 'delete',
ts: '2025-06-16T14:31:44.910Z',
userId: 'user-id',
},
},
],
},
123
)
).to.equal(false)
})
it('should handle docs above the limit and above with tracked-deletes removed', function () {
expect(
this.Limits.stringFileDataContentIsTooLarge(
{
content: 'x'.repeat(123 + 2),
trackedChanges: [
{
range: { pos: 1, length: 1 },
tracking: {
type: 'delete',
ts: '2025-06-16T14:31:44.910Z',
userId: 'user-id',
},
},
],
},
123
)
).to.equal(true)
})
it('should handle docs above the limit and with tracked-inserts', function () {
expect(
this.Limits.stringFileDataContentIsTooLarge(
{
content: 'x'.repeat(123 + 1),
trackedChanges: [
{
range: { pos: 1, length: 1 },
tracking: {
type: 'insert',
ts: '2025-06-16T14:31:44.910Z',
userId: 'user-id',
},
},
],
},
123
)
).to.equal(true)
})
})
})
services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js
+117 -49
View File
@@ -15,6 +15,7 @@ describe('ProjectHistoryRedisManager', function () {
this.Limits = {
docIsTooLarge: sinon.stub().returns(false),
stringFileDataContentIsTooLarge: sinon.stub().returns(false),
}
this.ProjectHistoryRedisManager = SandboxedModule.require(modulePath, {
@@ -61,22 +62,18 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should queue an update', function () {
this.multi.rpush
.calledWithExactly(
`ProjectHistory:Ops:${this.project_id}`,
this.ops[0],
this.ops[1]
)
.should.equal(true)
this.multi.rpush.should.have.been.calledWithExactly(
`ProjectHistory:Ops:${this.project_id}`,
this.ops[0],
this.ops[1]
)
})
it('should set the queue timestamp if not present', function () {
this.multi.setnx
.calledWithExactly(
`ProjectHistory:FirstOpTimestamp:${this.project_id}`,
Date.now()
)
.should.equal(true)
this.multi.setnx.should.have.been.calledWithExactly(
`ProjectHistory:FirstOpTimestamp:${this.project_id}`,
Date.now()
)
})
})
@@ -118,9 +115,10 @@ describe('ProjectHistoryRedisManager', function () {
file: this.file_id,
}
this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly(this.project_id, JSON.stringify(update))
.should.equal(true)
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
this.project_id,
JSON.stringify(update)
)
})
})
@@ -166,9 +164,10 @@ describe('ProjectHistoryRedisManager', function () {
doc: this.doc_id,
}
this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly(this.project_id, JSON.stringify(update))
.should.equal(true)
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
this.project_id,
JSON.stringify(update)
)
})
it('should queue an update with file metadata', async function () {
@@ -350,9 +349,10 @@ describe('ProjectHistoryRedisManager', function () {
doc: this.doc_id,
}
this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly(this.project_id, JSON.stringify(update))
.should.equal(true)
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
this.project_id,
JSON.stringify(update)
)
})
it('should not forward ranges if history ranges support is undefined', async function () {
@@ -402,9 +402,10 @@ describe('ProjectHistoryRedisManager', function () {
doc: this.doc_id,
}
this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly(this.project_id, JSON.stringify(update))
.should.equal(true)
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
this.project_id,
JSON.stringify(update)
)
})
it('should pass "false" as the createdBlob field if not provided', async function () {
@@ -432,9 +433,10 @@ describe('ProjectHistoryRedisManager', function () {
doc: this.doc_id,
}
this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly(this.project_id, JSON.stringify(update))
.should.equal(true)
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
this.project_id,
JSON.stringify(update)
)
})
it('should pass through the value of the createdBlob field', async function () {
@@ -463,9 +465,10 @@ describe('ProjectHistoryRedisManager', function () {
doc: this.doc_id,
}
this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly(this.project_id, JSON.stringify(update))
.should.equal(true)
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
this.project_id,
JSON.stringify(update)
)
})
})
@@ -493,8 +496,8 @@ describe('ProjectHistoryRedisManager', function () {
beforeEach(async function () {
this.update = {
resyncDocContent: {
content: 'one\ntwo',
version: this.version,
content: 'one\ntwo',
},
projectHistoryId: this.projectHistoryId,
path: this.pathname,
@@ -516,19 +519,18 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should check if the doc is too large', function () {
this.Limits.docIsTooLarge
.calledWith(
JSON.stringify(this.update).length,
this.lines,
this.settings.max_doc_length
)
.should.equal(true)
this.Limits.docIsTooLarge.should.have.been.calledWith(
JSON.stringify(this.update).length,
this.lines,
this.settings.max_doc_length
)
})
it('should queue an update', function () {
this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly(this.project_id, JSON.stringify(this.update))
.should.equal(true)
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
this.project_id,
JSON.stringify(this.update)
)
})
})
@@ -551,9 +553,8 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should not queue an update if the doc is too large', function () {
this.ProjectHistoryRedisManager.promises.queueOps.called.should.equal(
false
)
this.ProjectHistoryRedisManager.promises.queueOps.should.not.have.been
.called
})
})
@@ -561,10 +562,10 @@ describe('ProjectHistoryRedisManager', function () {
beforeEach(async function () {
this.update = {
resyncDocContent: {
content: 'onedeleted\ntwo',
version: this.version,
ranges: this.ranges,
resolvedCommentIds: this.resolvedCommentIds,
content: 'onedeleted\ntwo',
},
projectHistoryId: this.projectHistoryId,
path: this.pathname,
@@ -601,9 +602,76 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should queue an update', function () {
this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly(this.project_id, JSON.stringify(this.update))
.should.equal(true)
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
this.project_id,
JSON.stringify(this.update)
)
})
})
describe('history-ot', function () {
beforeEach(async function () {
this.lines = {
content: 'onedeleted\ntwo',
comments: [{ id: 'id1', ranges: [{ pos: 0, length: 3 }] }],
trackedChanges: [
{
range: { pos: 3, length: 7 },
tracking: {
type: 'delete',
userId: 'user-id',
ts: '2025-06-16T14:31:44.910Z',
},
},
],
}
this.update = {
resyncDocContent: {
version: this.version,
historyOTRanges: {
comments: this.lines.comments,
trackedChanges: this.lines.trackedChanges,
},
content: this.lines.content,
},
projectHistoryId: this.projectHistoryId,
path: this.pathname,
doc: this.doc_id,
meta: { ts: new Date() },
}
await this.ProjectHistoryRedisManager.promises.queueResyncDocContent(
this.project_id,
this.projectHistoryId,
this.doc_id,
this.lines,
this.ranges,
this.resolvedCommentIds,
this.version,
this.pathname,
true
)
})
it('should include tracked deletes in the update', function () {
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
this.project_id,
JSON.stringify(this.update)
)
})
it('should check the doc length without tracked deletes', function () {
this.Limits.stringFileDataContentIsTooLarge.should.have.been.calledWith(
this.lines,
this.settings.max_doc_length
)
})
it('should queue an update', function () {
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
this.project_id,
JSON.stringify(this.update)
)
})
})
})