agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-12 13:13:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

[monorepo] prepare per-project queues (#35386)

Browse Source 
GitOrigin-RevId: ab020603ec04c705d08be466563adac0ce30bc04
This commit is contained in:
Jakob Ackermann
2026-09-17 11:26:50 +00:00
committed by Copybot
parent 3700cf5968
commit cdba372dfc
23 changed files with 1583 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/real-time/app/js/DocumentUpdaterManager.js
+113 -2
View File
@@ -22,6 +22,15 @@ const {
const rclient = RedisWrapper.createClient(settings.redis.documentupdater)
const Keys = settings.redis.documentupdater.key_schema
/**
* Fetch a doc's lines, version, ranges and recent ops from the
* document-updater.
*
* @param {string} projectId
* @param {string} docId
* @param {number} fromVersion - return ops from this version onwards (-1 for
* no ops)
*/
async function getDocument(projectId, docId, fromVersion) {
const timer = new metrics.Timer('get-document')
const url = `${settings.apis.documentupdater.url}/project/${projectId}/doc/${docId}?fromVersion=${fromVersion}&historyOTSupport=true`
@@ -67,11 +76,23 @@ async function getDocument(projectId, docId, fromVersion) {
}
}
/**
* Check that a doc is loadable from the document-updater.
*
* @param {string} projectId
* @param {string} docId
*/
async function checkDocument(projectId, docId) {
// in this call fromVersion = -1 means get document without docOps
return await getDocument(projectId, docId, -1)
}
/**
* Ask the document-updater to flush a project to mongo and remove it from
* redis.
*
* @param {string} projectId
*/
async function flushProjectToMongoAndDelete(projectId) {
// this method is called when the last connected user leaves the project
logger.debug({ projectId }, 'deleting project from document updater')
@@ -98,6 +119,11 @@ async function flushProjectToMongoAndDelete(projectId) {
}
}
/**
* Pick a random shard of the document-updater dispatch list.
*
* @return {string}
*/
function _getPendingUpdateListKey() {
const shard = _.random(0, settings.pendingUpdateListShardCount - 1)
if (shard === 0) {
@@ -107,7 +133,22 @@ function _getPendingUpdateListKey() {
}
}
async function queueChange(projectId, docId, change) {
/**
* Queue an update for processing by the document-updater.
*
* @param {string} projectId
* @param {string} docId
* @param {Object} change
* @param {boolean} usePerProjectPendingUpdates - feature flag assigned on
* joinProject; in migration phase 2 it selects the per-project queue
* over the legacy per-doc queue
*/
async function queueChange(
projectId,
docId,
change,
usePerProjectPendingUpdates
) {
const allowedKeys = ['doc', 'op', 'v', 'dupIfSource', 'meta', 'lastV', 'hash']
change = _.pick(change, allowedKeys)
const jsonChange = JSON.stringify(change)
@@ -121,8 +162,44 @@ async function queueChange(projectId, docId, change) {
throw new UpdateTooLargeError(updateSize)
}
switch (settings.pendingUpdatesMigrationPhase) {
case 1:
await _queueChangeOnDocQueue(projectId, docId, jsonChange)
break
case 2:
// the consumer drains both queues in phase 2, so the feature flag can
// safely pick the queue per client
if (usePerProjectPendingUpdates) {
await _queueChangeOnProjectQueue(projectId, jsonChange)
} else {
await _queueChangeOnDocQueue(projectId, docId, jsonChange)
}
break
case 3:
await _queueChangeOnProjectQueue(projectId, jsonChange)
break
default:
throw new OError('invalid pendingUpdatesMigrationPhase', {
pendingUpdatesMigrationPhase: settings.pendingUpdatesMigrationPhase,
})
}
}
/**
* Queue an update on the legacy per-doc queue (phase 1): push the payload
* onto the doc queue, then put a `project_id:doc_id` marker onto
* pending-updates-list.
*
* @param {string} projectId
* @param {string} docId
* @param {string} jsonChange - the serialized change
*/
async function _queueChangeOnDocQueue(projectId, docId, jsonChange) {
// record metric for each update added to queue
metrics.summary('redis.pendingUpdates', updateSize, { status: 'push' })
metrics.summary('redis.pendingUpdates', jsonChange.length, {
status: 'push',
path: 'doc',
})
const docKey = `${projectId}:${docId}`
// Push onto pendingUpdates for doc_id first, because once the doc updater
@@ -143,6 +220,40 @@ async function queueChange(projectId, docId, change) {
}
}
/**
* Queue an update on the per-project queue (phases 2 & 3): push the payload
* (tagged with its doc id) onto the project queue, then put a bare
* `project_id` marker onto pending-updates-list.
*
* @param {string} projectId
* @param {string} jsonChange
*/
async function _queueChangeOnProjectQueue(projectId, jsonChange) {
// record metric for each update added to queue
metrics.summary('redis.pendingUpdates', jsonChange.length, {
status: 'push',
path: 'project',
})
try {
await rclient.rpush(
Keys.pendingProjectUpdates({ project_id: projectId }),
jsonChange
)
} catch (error) {
throw new OError('error pushing project update into redis').withCause(error)
}
const queueKey = _getPendingUpdateListKey()
try {
await rclient.rpush(queueKey, projectId)
} catch (error) {
throw new OError('error pushing project_id into redis')
.withInfo({ queueKey })
.withCause(error)
}
}
export default {
getDocument: callbackify(getDocument),
checkDocument: callbackify(checkDocument),
services/real-time/app/js/WebApiManager.js
+1
View File
@@ -54,6 +54,7 @@ async function joinProject(projectId, user) {
isRestrictedUser: data.isRestrictedUser,
isTokenMember: data.isTokenMember,
isInvitedMember: data.isInvitedMember,
usePerProjectPendingUpdates: data.usePerProjectPendingUpdates,
}
return {
project: data.project,
services/real-time/app/js/WebsocketController.js
+8 -1
View File
@@ -100,6 +100,8 @@ export default WebsocketController = {
client.ol_context.is_restricted_user = !!userMetadata.isRestrictedUser
client.ol_context.is_token_member = !!userMetadata.isTokenMember
client.ol_context.is_invited_member = !!userMetadata.isInvitedMember
client.ol_context.use_per_project_pending_updates =
!!userMetadata.usePerProjectPendingUpdates
RoomManager.joinProject(client, projectId, function (err) {
if (err) {
@@ -557,7 +559,11 @@ export default WebsocketController = {
applyOtUpdate(client, docId, update, callback) {
// client may have disconnected, but we can submit their update to doc-updater anyways.
const { user_id: userId, project_id: projectId } = client.ol_context
const {
user_id: userId,
project_id: projectId,
use_per_project_pending_updates: usePerProjectPendingUpdates,
} = client.ol_context
if (!projectId) {
return callback(new NotJoinedError())
}
@@ -607,6 +613,7 @@ export default WebsocketController = {
projectId,
docId,
update,
usePerProjectPendingUpdates,
function (error) {
if ((error && error.message) === 'update is too large') {
metrics.inc('update_too_large')
services/real-time/config/settings.defaults.cjs
+17
View File
@@ -64,6 +64,9 @@ const settings = {
pendingUpdates({ doc_id }) {
return `PendingUpdates:{${doc_id}}`
},
pendingProjectUpdates({ project_id }) {
return `PendingProjectUpdates:{${project_id}}`
},
},
maxRetriesPerRequest: parseInt(
process.env.DOC_UPDATER_REDIS_MAX_RETRIES_PER_REQUEST ||
@@ -131,6 +134,20 @@ const settings = {
10
),
// Migration of the document-updater update queue from a per-doc queue to a
// per-project queue. Must be kept in step with document-updater.
// phase 1: write the legacy per-doc queue (`PendingUpdates:{doc_id}` +
// `<project_id>:<doc_id>` marker)
// phase 2: write the per-project queue (`PendingProjectUpdates:{project_id}`
// + bare `<project_id>` marker) for clients with the
// per-project-pending-updates feature flag (assigned by web on
// joinProject), the legacy per-doc queue otherwise
// phase 3: write the per-project queue for all clients
pendingUpdatesMigrationPhase: parseInt(
process.env.PENDING_UPDATES_MIGRATION_PHASE || '1',
10
),
// combine
// max_doc_length (2mb see above) * 2 (delete + insert)
// max_ranges_size (3mb see MAX_RANGES_SIZE in document-updater)
services/real-time/test/acceptance/js/helpers/MockWebServer.js
+2
View File
@@ -74,6 +74,8 @@ export default MockWebServer = {
isRestrictedUser: !!userMetadata?.isRestrictedUser,
isTokenMember: !!userMetadata?.isTokenMember,
isInvitedMember: !!userMetadata?.isInvitedMember,
usePerProjectPendingUpdates:
!!userMetadata?.usePerProjectPendingUpdates,
})
}
)
services/real-time/test/unit/js/DocumentUpdaterManager.test.js
+102 -1
View File
@@ -24,11 +24,15 @@ describe('DocumentUpdaterManager', function () {
pendingUpdates({ doc_id: docId }) {
return `PendingUpdates:${docId}`
},
pendingProjectUpdates({ project_id: projectId }) {
return `PendingProjectUpdates:${projectId}`
},
},
},
},
maxUpdateSize: 7 * 1024 * 1024,
pendingUpdateListShardCount: 10,
pendingUpdatesMigrationPhase: 1,
}
ctx.rclient = { auth() {} }
ctx.fetchJson = sinon.stub()
@@ -239,7 +243,7 @@ describe('DocumentUpdaterManager', function () {
describe('queueChange', function () {
beforeEach(function (ctx) {
ctx.change = {
doc: '1234567890',
doc: ctx.doc_id,
op: [{ d: 'test', p: 345 }],
v: 789,
}
@@ -274,6 +278,103 @@ describe('DocumentUpdaterManager', function () {
})
})
describe('in migration phase 2 with the per-project queue flag', function () {
beforeEach(async function (ctx) {
ctx.settings.pendingUpdatesMigrationPhase = 2
await ctx.DocumentUpdaterManager.promises.queueChange(
ctx.project_id,
ctx.doc_id,
ctx.change,
true
)
})
it('should push the change onto the per-project queue with its doc id', function (ctx) {
ctx.rclient.rpush
.calledWith(
`PendingProjectUpdates:${ctx.project_id}`,
JSON.stringify(ctx.change)
)
.should.equal(true)
})
it('should not push onto the legacy per-doc queue', function (ctx) {
ctx.rclient.rpush
.calledWith(`PendingUpdates:${ctx.doc_id}`)
.should.equal(false)
})
it('should notify the doc updater with a bare project id marker', function (ctx) {
const secondCall = ctx.rclient.rpush.secondCall
secondCall.should.exist
const queueKey = secondCall.args[0]
queueKey.should.match(/^pending-updates-list(-\d+)?$/)
secondCall.args[1].should.equal(ctx.project_id)
})
})
describe('in migration phase 2 without the per-project queue flag', function () {
beforeEach(async function (ctx) {
ctx.settings.pendingUpdatesMigrationPhase = 2
await ctx.DocumentUpdaterManager.promises.queueChange(
ctx.project_id,
ctx.doc_id,
ctx.change,
false
)
})
it('should push the change onto the legacy per-doc queue', function (ctx) {
ctx.rclient.rpush
.calledWith(
`PendingUpdates:${ctx.doc_id}`,
JSON.stringify(ctx.change)
)
.should.equal(true)
})
it('should not push onto the per-project queue', function (ctx) {
ctx.rclient.rpush
.calledWith(`PendingProjectUpdates:${ctx.project_id}`)
.should.equal(false)
})
it('should notify the doc updater with a project_id:doc_id marker', function (ctx) {
const secondCall = ctx.rclient.rpush.secondCall
secondCall.should.exist
const queueKey = secondCall.args[0]
queueKey.should.match(/^pending-updates-list(-\d+)?$/)
secondCall.args[1].should.equal(`${ctx.project_id}:${ctx.doc_id}`)
})
})
describe('in migration phase 3 (per-project queue only)', function () {
beforeEach(async function (ctx) {
ctx.settings.pendingUpdatesMigrationPhase = 3
await ctx.DocumentUpdaterManager.promises.queueChange(
ctx.project_id,
ctx.doc_id,
ctx.change,
false
)
})
it('should push the change onto the per-project queue even without the flag', function (ctx) {
ctx.rclient.rpush
.calledWith(
`PendingProjectUpdates:${ctx.project_id}`,
JSON.stringify(ctx.change)
)
.should.equal(true)
})
it('should not push onto the legacy per-doc queue', function (ctx) {
ctx.rclient.rpush
.calledWith(`PendingUpdates:${ctx.doc_id}`)
.should.equal(false)
})
})
describe('with error talking to redis during rpush', function () {
beforeEach(function (ctx) {
ctx.rclient.rpush = sinon
services/real-time/test/unit/js/WebApiManager.test.js
+6
View File
@@ -42,6 +42,7 @@ describe('WebApiManager', function () {
isRestrictedUser: true,
isTokenMember: true,
isInvitedMember: true,
usePerProjectPendingUpdates: true,
}
ctx.fetchUtils.fetchJson.resolves(ctx.response)
ctx.WebApiManager.joinProject(ctx.project_id, ctx.user, ctx.callback)
@@ -70,6 +71,8 @@ describe('WebApiManager', function () {
isRestrictedUser: ctx.response.isRestrictedUser,
isTokenMember: ctx.response.isTokenMember,
isInvitedMember: ctx.response.isInvitedMember,
usePerProjectPendingUpdates:
ctx.response.usePerProjectPendingUpdates,
})
.should.equal(true)
})
@@ -89,6 +92,7 @@ describe('WebApiManager', function () {
isRestrictedUser: true,
isTokenMember: false,
isInvitedMember: false,
usePerProjectPendingUpdates: false,
}
ctx.fetchUtils.fetchJson.resolves(ctx.response)
ctx.WebApiManager.joinProject(ctx.project_id, ctx.user, ctx.callback)
@@ -120,6 +124,8 @@ describe('WebApiManager', function () {
isRestrictedUser: ctx.response.isRestrictedUser,
isTokenMember: ctx.response.isTokenMember,
isInvitedMember: ctx.response.isInvitedMember,
usePerProjectPendingUpdates:
ctx.response.usePerProjectPendingUpdates,
}
)
})
services/real-time/test/unit/js/WebsocketController.test.js
+15 -5
View File
@@ -97,12 +97,14 @@ describe('WebsocketController', function () {
ctx.isRestrictedUser = true
ctx.isTokenMember = true
ctx.isInvitedMember = true
ctx.usePerProjectPendingUpdates = true
ctx.WebApiManager.joinProject = sinon
.stub()
.callsArgWith(2, null, ctx.project, ctx.privilegeLevel, {
isRestrictedUser: ctx.isRestrictedUser,
isTokenMember: ctx.isTokenMember,
isInvitedMember: ctx.isInvitedMember,
usePerProjectPendingUpdates: ctx.usePerProjectPendingUpdates,
})
ctx.RoomManager.joinProject = sinon.stub().callsArg(2)
return ctx.WebsocketController.joinProject(
@@ -176,6 +178,11 @@ describe('WebsocketController', function () {
ctx.isInvitedMember
)
})
it('should set the use_per_project_pending_updates flag on the client', function (ctx) {
ctx.client.ol_context.use_per_project_pending_updates.should.equal(
ctx.usePerProjectPendingUpdates
)
})
it('should call the callback with the project, privilegeLevel and protocolVersion', function (ctx) {
return ctx.callback
.calledWith(
@@ -240,12 +247,14 @@ describe('WebsocketController', function () {
ctx.isRestrictedUser = true
ctx.isTokenMember = true
ctx.isInvitedMember = true
ctx.usePerProjectPendingUpdates = true
ctx.WebApiManager.joinProject = sinon
.stub()
.callsArgWith(2, null, ctx.project, ctx.privilegeLevel, {
isRestrictedUser: ctx.isRestrictedUser,
isTokenMember: ctx.isTokenMember,
isInvitedMember: ctx.isInvitedMember,
usePerProjectPendingUpdates: ctx.usePerProjectPendingUpdates,
})
ctx.RoomManager.joinProject = sinon
.stub()
@@ -1466,10 +1475,11 @@ describe('WebsocketController', function () {
ctx.update = { op: { p: 12, t: 'foo' } }
ctx.client.ol_context.user_id = ctx.user_id
ctx.client.ol_context.project_id = ctx.project_id
ctx.client.ol_context.use_per_project_pending_updates = true
ctx.WebsocketController._assertClientCanApplyUpdate = sinon
.stub()
.yields()
ctx.DocumentUpdaterManager.queueChange = sinon.stub().callsArg(3)
ctx.DocumentUpdaterManager.queueChange = sinon.stub().callsArg(4)
})
describe('succesfully', function () {
@@ -1490,9 +1500,9 @@ describe('WebsocketController', function () {
ctx.update.meta.user_id.should.equal(ctx.user_id)
})
it('should queue the update', function (ctx) {
it('should queue the update with the per-project queue flag', function (ctx) {
ctx.DocumentUpdaterManager.queueChange
.calledWith(ctx.project_id, ctx.doc_id, ctx.update)
.calledWith(ctx.project_id, ctx.doc_id, ctx.update, true)
.should.equal(true)
})
@@ -1510,7 +1520,7 @@ describe('WebsocketController', function () {
ctx.client.disconnect = sinon.stub()
ctx.DocumentUpdaterManager.queueChange = sinon
.stub()
.callsArgWith(3, (ctx.error = new Error('Something went wrong')))
.callsArgWith(4, (ctx.error = new Error('Something went wrong')))
ctx.WebsocketController.applyOtUpdate(
ctx.client,
ctx.doc_id,
@@ -1570,7 +1580,7 @@ describe('WebsocketController', function () {
const error = new UpdateTooLargeError(7372835)
ctx.DocumentUpdaterManager.queueChange = sinon
.stub()
.callsArgWith(3, error)
.callsArgWith(4, error)
ctx.WebsocketController.applyOtUpdate(
ctx.client,
ctx.doc_id,