mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-14 14:13:43 +01:00
[monorepo] prepare per-project queues (#35386)
GitOrigin-RevId: ab020603ec04c705d08be466563adac0ce30bc04
This commit is contained in:
Jakob Ackermann
@@ -36,6 +36,7 @@ describe('UpdateManager', function () {
|
||||
promises: {
|
||||
tryLock: sinon.stub().resolves(this.projectLockValue),
|
||||
getLock: sinon.stub().resolves(this.projectLockValue),
|
||||
extendLock: sinon.stub().resolves(),
|
||||
releaseLock: sinon.stub().resolves(),
|
||||
},
|
||||
}
|
||||
@@ -45,17 +46,25 @@ describe('UpdateManager', function () {
|
||||
setDocument: sinon.stub().resolves(),
|
||||
updateDocument: sinon.stub(),
|
||||
recordProjectNotificationTimestamp: sinon.stub().resolves(),
|
||||
getDocIdsInProject: sinon.stub().resolves([]),
|
||||
},
|
||||
}
|
||||
|
||||
this.RealTimeRedisManager = {
|
||||
sendData: sinon.stub(),
|
||||
promises: {
|
||||
getUpdatesLength: sinon.stub(),
|
||||
getUpdatesLength: sinon.stub().resolves(0),
|
||||
getPendingUpdatesForDoc: sinon.stub(),
|
||||
getProjectUpdatesLength: sinon.stub().resolves(0),
|
||||
getPendingProjectUpdates: sinon.stub().resolves([]),
|
||||
},
|
||||
}
|
||||
|
||||
this.HistoryOTUpdateManager = {
|
||||
isHistoryOTEditOperationUpdate: sinon.stub().returns(false),
|
||||
applyUpdate: sinon.stub().resolves(),
|
||||
}
|
||||
|
||||
this.ShareJsUpdateManager = {
|
||||
promises: {
|
||||
applyUpdate: sinon.stub(),
|
||||
@@ -66,7 +75,7 @@ describe('UpdateManager', function () {
|
||||
recordAndFlushHistoryOps: sinon.stub(),
|
||||
}
|
||||
|
||||
this.Settings = {}
|
||||
this.Settings = { pendingUpdatesMigrationPhase: 1 }
|
||||
|
||||
this.DocumentManager = {
|
||||
promises: {
|
||||
@@ -106,6 +115,7 @@ describe('UpdateManager', function () {
|
||||
'./RealTimeRedisManager': this.RealTimeRedisManager,
|
||||
'./ShareJsUpdateManager': this.ShareJsUpdateManager,
|
||||
'./HistoryManager': this.HistoryManager,
|
||||
'./HistoryOTUpdateManager': this.HistoryOTUpdateManager,
|
||||
'./Metrics': this.Metrics,
|
||||
'@overleaf/settings': this.Settings,
|
||||
'./DocumentManager': this.DocumentManager,
|
||||
@@ -139,12 +149,12 @@ describe('UpdateManager', function () {
|
||||
})
|
||||
|
||||
describe('processOutstandingUpdatesWithLock', function () {
|
||||
describe('when the lock is free', function () {
|
||||
describe('when the project lock is free', function () {
|
||||
beforeEach(function () {
|
||||
this.UpdateManager.promises.continueProcessingUpdatesWithLock = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
this.UpdateManager.promises.processOutstandingUpdates = sinon
|
||||
this.UpdateManager.promises.processOutstandingProjectUpdates = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
})
|
||||
@@ -157,25 +167,10 @@ describe('UpdateManager', function () {
|
||||
)
|
||||
})
|
||||
|
||||
it('should acquire the project lock before the doc lock', function () {
|
||||
it('should acquire the project lock', function () {
|
||||
this.ProjectLockManager.promises.getLock
|
||||
.calledWith(this.project_id)
|
||||
.should.equal(true)
|
||||
this.LockManager.promises.tryLock
|
||||
.calledAfter(this.ProjectLockManager.promises.getLock)
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
it('should acquire the lock', function () {
|
||||
this.LockManager.promises.tryLock
|
||||
.calledWith(this.doc_id)
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
it('should free the lock', function () {
|
||||
this.LockManager.promises.releaseLock
|
||||
.calledWith(this.doc_id, this.lockValue)
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
it('should free the project lock', function () {
|
||||
@@ -184,18 +179,18 @@ describe('UpdateManager', function () {
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
it('should process the outstanding updates', function () {
|
||||
this.UpdateManager.promises.processOutstandingUpdates
|
||||
.calledWith(this.project_id, this.doc_id)
|
||||
it('should process the outstanding project updates', function () {
|
||||
this.UpdateManager.promises.processOutstandingProjectUpdates
|
||||
.calledWith(this.project_id, this.doc_id, this.projectLockValue)
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
it('should do everything with the lock acquired', function () {
|
||||
this.UpdateManager.promises.processOutstandingUpdates
|
||||
.calledAfter(this.LockManager.promises.tryLock)
|
||||
this.UpdateManager.promises.processOutstandingProjectUpdates
|
||||
.calledAfter(this.ProjectLockManager.promises.getLock)
|
||||
.should.equal(true)
|
||||
this.UpdateManager.promises.processOutstandingUpdates
|
||||
.calledBefore(this.LockManager.promises.releaseLock)
|
||||
this.UpdateManager.promises.processOutstandingProjectUpdates
|
||||
.calledBefore(this.ProjectLockManager.promises.releaseLock)
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
@@ -206,10 +201,10 @@ describe('UpdateManager', function () {
|
||||
})
|
||||
})
|
||||
|
||||
describe('when processOutstandingUpdates returns an error', function () {
|
||||
describe('when processOutstandingProjectUpdates returns an error', function () {
|
||||
beforeEach(async function () {
|
||||
this.error = new Error('Something went wrong')
|
||||
this.UpdateManager.promises.processOutstandingUpdates = sinon
|
||||
this.UpdateManager.promises.processOutstandingProjectUpdates = sinon
|
||||
.stub()
|
||||
.rejects(this.error)
|
||||
await expect(
|
||||
@@ -220,12 +215,6 @@ describe('UpdateManager', function () {
|
||||
).to.be.rejectedWith(this.error)
|
||||
})
|
||||
|
||||
it('should free the lock', function () {
|
||||
this.LockManager.promises.releaseLock
|
||||
.calledWith(this.doc_id, this.lockValue)
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
it('should free the project lock', function () {
|
||||
this.ProjectLockManager.promises.releaseLock
|
||||
.calledWith(this.project_id, this.projectLockValue)
|
||||
@@ -233,37 +222,247 @@ describe('UpdateManager', function () {
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the lock is taken', function () {
|
||||
describe('processOutstandingProjectUpdates', function () {
|
||||
beforeEach(function () {
|
||||
this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
this.UpdateManager.promises.fetchAndApplyProjectUpdates = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
})
|
||||
|
||||
describe('in phases 1 & 2', function () {
|
||||
beforeEach(async function () {
|
||||
this.LockManager.promises.tryLock.resolves(null)
|
||||
this.UpdateManager.promises.processOutstandingUpdates = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
await this.UpdateManager.promises.processOutstandingUpdatesWithLock(
|
||||
this.Settings.pendingUpdatesMigrationPhase = 2
|
||||
this.RedisManager.promises.getDocIdsInProject.resolves([
|
||||
'doc-1',
|
||||
'doc-2',
|
||||
])
|
||||
await this.UpdateManager.promises.processOutstandingProjectUpdates(
|
||||
this.project_id,
|
||||
this.doc_id,
|
||||
this.projectLockValue,
|
||||
new this.Profiler()
|
||||
)
|
||||
})
|
||||
|
||||
it('should drain the per-doc queue for every doc in the project plus the hint', function () {
|
||||
this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock.should.have.been.calledWith(
|
||||
this.project_id,
|
||||
'doc-1'
|
||||
)
|
||||
this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock.should.have.been.calledWith(
|
||||
this.project_id,
|
||||
'doc-2'
|
||||
)
|
||||
this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock.should.have.been.calledWith(
|
||||
this.project_id,
|
||||
this.doc_id
|
||||
)
|
||||
})
|
||||
|
||||
it('should not process the updates', function () {
|
||||
this.UpdateManager.promises.processOutstandingUpdates.called.should.equal(
|
||||
it('should extend the project lock between docs', function () {
|
||||
this.ProjectLockManager.promises.extendLock.should.have.been.calledWith(
|
||||
this.project_id,
|
||||
this.projectLockValue
|
||||
)
|
||||
})
|
||||
|
||||
it('should drain the per-project queue', function () {
|
||||
this.UpdateManager.promises.fetchAndApplyProjectUpdates.should.have.been.calledWith(
|
||||
this.project_id
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('in phase 3', function () {
|
||||
beforeEach(async function () {
|
||||
this.Settings.pendingUpdatesMigrationPhase = 3
|
||||
await this.UpdateManager.promises.processOutstandingProjectUpdates(
|
||||
this.project_id,
|
||||
this.doc_id,
|
||||
this.projectLockValue,
|
||||
new this.Profiler()
|
||||
)
|
||||
})
|
||||
|
||||
it('should not look up the docs in the project', function () {
|
||||
this.RedisManager.promises.getDocIdsInProject.called.should.equal(false)
|
||||
})
|
||||
|
||||
it('should not drain any per-doc queue', function () {
|
||||
this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock.called.should.equal(
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
it('should free the project lock', function () {
|
||||
this.ProjectLockManager.promises.releaseLock
|
||||
.calledWith(this.project_id, this.projectLockValue)
|
||||
it('should not bump the project lock (freshly acquired, no per-doc loop)', function () {
|
||||
this.ProjectLockManager.promises.extendLock.called.should.equal(false)
|
||||
})
|
||||
|
||||
it('should drain the per-project queue', function () {
|
||||
this.UpdateManager.promises.fetchAndApplyProjectUpdates.should.have.been.calledWith(
|
||||
this.project_id
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('fetchAndApplyDocUpdatesUnderDocLock', function () {
|
||||
beforeEach(function () {
|
||||
this.UpdateManager.promises.processOutstandingUpdates = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
})
|
||||
|
||||
describe('when the doc has pending updates', function () {
|
||||
beforeEach(async function () {
|
||||
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(2)
|
||||
await this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock(
|
||||
this.project_id,
|
||||
this.doc_id,
|
||||
new this.Profiler()
|
||||
)
|
||||
})
|
||||
|
||||
it('should take the per-doc lock', function () {
|
||||
this.LockManager.promises.tryLock
|
||||
.calledWith(this.doc_id)
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
it('should process the doc updates and release the per-doc lock', function () {
|
||||
this.UpdateManager.promises.processOutstandingUpdates.should.have.been.calledWith(
|
||||
this.project_id,
|
||||
this.doc_id
|
||||
)
|
||||
this.LockManager.promises.releaseLock
|
||||
.calledWith(this.doc_id, this.lockValue)
|
||||
.should.equal(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the doc has no pending updates', function () {
|
||||
beforeEach(async function () {
|
||||
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(0)
|
||||
await this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock(
|
||||
this.project_id,
|
||||
this.doc_id,
|
||||
new this.Profiler()
|
||||
)
|
||||
})
|
||||
|
||||
it('should not take the per-doc lock', function () {
|
||||
this.LockManager.promises.tryLock.called.should.equal(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the per-doc lock is taken (e.g. by an old instance)', function () {
|
||||
beforeEach(async function () {
|
||||
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(2)
|
||||
this.LockManager.promises.tryLock.resolves(null)
|
||||
await this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock(
|
||||
this.project_id,
|
||||
this.doc_id,
|
||||
new this.Profiler()
|
||||
)
|
||||
})
|
||||
|
||||
it('should skip processing the doc', function () {
|
||||
this.UpdateManager.promises.processOutstandingUpdates.called.should.equal(
|
||||
false
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('fetchAndApplyProjectUpdates', function () {
|
||||
beforeEach(function () {
|
||||
this.UpdateManager.promises.applyUpdate = sinon.stub().resolves()
|
||||
})
|
||||
|
||||
describe('with updates carrying their doc id', function () {
|
||||
beforeEach(async function () {
|
||||
this.updates = [
|
||||
{ doc: 'doc-1', op: 'a' },
|
||||
{ doc: 'doc-2', op: 'b' },
|
||||
]
|
||||
this.RealTimeRedisManager.promises.getPendingProjectUpdates.resolves(
|
||||
this.updates
|
||||
)
|
||||
await this.UpdateManager.promises.fetchAndApplyProjectUpdates(
|
||||
this.project_id,
|
||||
new this.Profiler()
|
||||
)
|
||||
})
|
||||
|
||||
it('should read the per-project queue', function () {
|
||||
this.RealTimeRedisManager.promises.getPendingProjectUpdates
|
||||
.calledWith(this.project_id)
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
it('should apply each update to the doc it carries', function () {
|
||||
this.UpdateManager.promises.applyUpdate.should.have.been.calledWith(
|
||||
this.project_id,
|
||||
'doc-1',
|
||||
this.updates[0]
|
||||
)
|
||||
this.UpdateManager.promises.applyUpdate.should.have.been.calledWith(
|
||||
this.project_id,
|
||||
'doc-2',
|
||||
this.updates[1]
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('with a history-OT update', function () {
|
||||
beforeEach(async function () {
|
||||
this.update = { doc: 'doc-1', op: 'a' }
|
||||
this.RealTimeRedisManager.promises.getPendingProjectUpdates.resolves([
|
||||
this.update,
|
||||
])
|
||||
this.HistoryOTUpdateManager.isHistoryOTEditOperationUpdate.returns(true)
|
||||
await this.UpdateManager.promises.fetchAndApplyProjectUpdates(
|
||||
this.project_id,
|
||||
new this.Profiler()
|
||||
)
|
||||
})
|
||||
|
||||
it('should route it to the history-OT update manager', function () {
|
||||
this.HistoryOTUpdateManager.applyUpdate.should.have.been.calledWith(
|
||||
this.project_id,
|
||||
'doc-1',
|
||||
this.update
|
||||
)
|
||||
this.UpdateManager.promises.applyUpdate.called.should.equal(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('with no updates', function () {
|
||||
beforeEach(async function () {
|
||||
this.RealTimeRedisManager.promises.getPendingProjectUpdates.resolves([])
|
||||
await this.UpdateManager.promises.fetchAndApplyProjectUpdates(
|
||||
this.project_id,
|
||||
new this.Profiler()
|
||||
)
|
||||
})
|
||||
|
||||
it('should not apply any update', function () {
|
||||
this.UpdateManager.promises.applyUpdate.called.should.equal(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('continueProcessingUpdatesWithLock', function () {
|
||||
describe('when there are outstanding updates', function () {
|
||||
beforeEach(async function () {
|
||||
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(3)
|
||||
this.UpdateManager.promises.getProjectPendingUpdatesLength = sinon
|
||||
.stub()
|
||||
.resolves(3)
|
||||
this.UpdateManager.promises.processOutstandingUpdatesWithLock = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
@@ -282,7 +481,9 @@ describe('UpdateManager', function () {
|
||||
|
||||
describe('when there are no outstanding updates', function () {
|
||||
beforeEach(async function () {
|
||||
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(0)
|
||||
this.UpdateManager.promises.getProjectPendingUpdatesLength = sinon
|
||||
.stub()
|
||||
.resolves(0)
|
||||
this.UpdateManager.promises.processOutstandingUpdatesWithLock = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
@@ -300,6 +501,61 @@ describe('UpdateManager', function () {
|
||||
})
|
||||
})
|
||||
|
||||
describe('getProjectPendingUpdatesLength', function () {
|
||||
describe('when the per-project queue is non-empty', function () {
|
||||
beforeEach(async function () {
|
||||
this.RealTimeRedisManager.promises.getProjectUpdatesLength.resolves(4)
|
||||
this.length =
|
||||
await this.UpdateManager.promises.getProjectPendingUpdatesLength(
|
||||
this.project_id,
|
||||
this.doc_id
|
||||
)
|
||||
})
|
||||
|
||||
it('should short-circuit without scanning per-doc queues', function () {
|
||||
this.length.should.equal(4)
|
||||
this.RedisManager.promises.getDocIdsInProject.called.should.equal(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the per-project queue is empty (phases 1 & 2)', function () {
|
||||
beforeEach(async function () {
|
||||
this.Settings.pendingUpdatesMigrationPhase = 2
|
||||
this.RealTimeRedisManager.promises.getProjectUpdatesLength.resolves(0)
|
||||
this.RedisManager.promises.getDocIdsInProject.resolves(['doc-1'])
|
||||
this.RealTimeRedisManager.promises.getUpdatesLength
|
||||
.withArgs('doc-1')
|
||||
.resolves(2)
|
||||
this.length =
|
||||
await this.UpdateManager.promises.getProjectPendingUpdatesLength(
|
||||
this.project_id,
|
||||
this.doc_id
|
||||
)
|
||||
})
|
||||
|
||||
it('should fall back to the per-doc queues', function () {
|
||||
this.length.should.equal(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('in phase 3', function () {
|
||||
beforeEach(async function () {
|
||||
this.Settings.pendingUpdatesMigrationPhase = 3
|
||||
this.RealTimeRedisManager.promises.getProjectUpdatesLength.resolves(0)
|
||||
this.length =
|
||||
await this.UpdateManager.promises.getProjectPendingUpdatesLength(
|
||||
this.project_id,
|
||||
this.doc_id
|
||||
)
|
||||
})
|
||||
|
||||
it('should only consider the per-project queue', function () {
|
||||
this.length.should.equal(0)
|
||||
this.RedisManager.promises.getDocIdsInProject.called.should.equal(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('fetchAndApplyUpdates', function () {
|
||||
describe('with updates', function () {
|
||||
beforeEach(async function () {
|
||||
@@ -902,7 +1158,7 @@ describe('UpdateManager', function () {
|
||||
this.UpdateManager.promises.continueProcessingUpdatesWithLock = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
this.UpdateManager.promises.processOutstandingUpdates = sinon
|
||||
this.UpdateManager.promises.processOutstandingProjectUpdates = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
this.response = await this.UpdateManager.promises.lockUpdatesAndDo(
|
||||
@@ -925,10 +1181,11 @@ describe('UpdateManager', function () {
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
it('should process any outstanding updates', function () {
|
||||
this.UpdateManager.promises.processOutstandingUpdates.should.have.been.calledWith(
|
||||
it('should process any outstanding updates for the project', function () {
|
||||
this.UpdateManager.promises.processOutstandingProjectUpdates.should.have.been.calledWith(
|
||||
this.project_id,
|
||||
this.doc_id
|
||||
this.doc_id,
|
||||
this.projectLockValue
|
||||
)
|
||||
})
|
||||
|
||||
@@ -958,10 +1215,10 @@ describe('UpdateManager', function () {
|
||||
})
|
||||
})
|
||||
|
||||
describe('when processOutstandingUpdates returns an error', function () {
|
||||
describe('when processOutstandingProjectUpdates returns an error', function () {
|
||||
beforeEach(async function () {
|
||||
this.error = new Error('Something went wrong')
|
||||
this.UpdateManager.promises.processOutstandingUpdates = sinon
|
||||
this.UpdateManager.promises.processOutstandingProjectUpdates = sinon
|
||||
.stub()
|
||||
.rejects(this.error)
|
||||
await expect(
|
||||
@@ -974,10 +1231,7 @@ describe('UpdateManager', function () {
|
||||
).to.be.rejectedWith(this.error)
|
||||
})
|
||||
|
||||
it('should free both locks', function () {
|
||||
this.LockManager.promises.releaseLock
|
||||
.calledWith(this.doc_id, this.lockValue)
|
||||
.should.equal(true)
|
||||
it('should free the project lock', function () {
|
||||
this.ProjectLockManager.promises.releaseLock
|
||||
.calledWith(this.project_id, this.projectLockValue)
|
||||
.should.equal(true)
|
||||
@@ -987,7 +1241,7 @@ describe('UpdateManager', function () {
|
||||
describe('when the method returns an error', function () {
|
||||
beforeEach(async function () {
|
||||
this.error = new Error('something went wrong')
|
||||
this.UpdateManager.promises.processOutstandingUpdates = sinon
|
||||
this.UpdateManager.promises.processOutstandingProjectUpdates = sinon
|
||||
.stub()
|
||||
.resolves()
|
||||
this.method = sinon.stub().rejects(this.error)
|
||||
|
||||
Reference in New Issue
Block a user