agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-12 21:23:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

[monorepo] prepare per-project queues (#35386)

Browse Source 
GitOrigin-RevId: ab020603ec04c705d08be466563adac0ce30bc04
This commit is contained in:
Jakob Ackermann
2026-09-17 11:26:50 +00:00
committed by Copybot
parent 3700cf5968
commit cdba372dfc
23 changed files with 1583 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/document-updater/app/js/RealTimeRedisManager.js
+105 -43
View File
@@ -30,59 +30,116 @@ let COUNT = 0
const MAX_OPS_PER_ITERATION = 8 // process a limited number of ops for safety
/**
* Shared implementation for draining a batch of updates from a
* pending-updates list. Used for both the per-doc and per-project queues;
* `path` ('doc' | 'project') differentiates the two in metrics.
*
* The MULTI only operates on the single passed key (which has an id in curly
* braces), so all of its operations run on the same node in a cluster
* environment.
*
* @param {string} key - the redis key of the queue
* @param {'doc' | 'project'} path - metrics label
* @param {function(Error, Array<Object>=): void} callback
*/
function getPendingUpdatesFromQueue(key, path, callback) {
const multi = rclient.multi()
multi.llen(key)
multi.lrange(key, 0, MAX_OPS_PER_ITERATION - 1)
multi.ltrim(key, MAX_OPS_PER_ITERATION, -1)
multi.exec(function (error, replys) {
if (error != null) {
return callback(error)
}
const [llen, jsonUpdates, _trimResult] = replys
metrics.histogram(
'redis.pendingUpdates.llen',
llen,
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 50, 75, 100],
{ path }
)
for (const jsonUpdate of jsonUpdates) {
// record metric for each update removed from queue
metrics.summary('redis.pendingUpdates', jsonUpdate.length, {
status: 'pop',
path,
})
}
const updates = []
for (const jsonUpdate of jsonUpdates) {
let update
try {
update = JSON.parse(jsonUpdate)
} catch (e) {
return callback(e)
}
updates.push(update)
}
return callback(error, updates)
})
}
const RealTimeRedisManager = {
/**
* Drain a batch of updates from a doc's legacy per-doc queue.
*
* @param {string} docId
* @param {function(Error, Array<Object>=): void} callback
*/
getPendingUpdatesForDoc(docId, callback) {
// Make sure that this MULTI operation only operates on doc
// specific keys, i.e. keys that have the doc id in curly braces.
// The curly braces identify a hash key for Redis and ensures that
// the MULTI's operations are all done on the same node in a
// cluster environment.
const multi = rclient.multi()
multi.llen(Keys.pendingUpdates({ doc_id: docId }))
multi.lrange(
getPendingUpdatesFromQueue(
Keys.pendingUpdates({ doc_id: docId }),
0,
MAX_OPS_PER_ITERATION - 1
'doc',
callback
)
multi.ltrim(
Keys.pendingUpdates({ doc_id: docId }),
MAX_OPS_PER_ITERATION,
-1
)
multi.exec(function (error, replys) {
if (error != null) {
return callback(error)
}
const [llen, jsonUpdates, _trimResult] = replys
metrics.histogram(
'redis.pendingUpdates.llen',
llen,
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 50, 75, 100]
)
for (const jsonUpdate of jsonUpdates) {
// record metric for each update removed from queue
metrics.summary('redis.pendingUpdates', jsonUpdate.length, {
status: 'pop',
})
}
const updates = []
for (const jsonUpdate of jsonUpdates) {
let update
try {
update = JSON.parse(jsonUpdate)
} catch (e) {
return callback(e)
}
updates.push(update)
}
return callback(error, updates)
})
},
/**
* Get the length of a doc's legacy per-doc queue.
*
* @param {string} docId
* @param {function(Error, number=): void} callback
*/
getUpdatesLength(docId, callback) {
rclient.llen(Keys.pendingUpdates({ doc_id: docId }), callback)
},
/**
* Drain a batch of updates from a project's per-project queue. Each update
* on the per-project queue carries its own `doc` id so the caller knows
* which doc to apply it to.
*
* @param {string} projectId
* @param {function(Error, Array<Object>=): void} callback
*/
getPendingProjectUpdates(projectId, callback) {
getPendingUpdatesFromQueue(
Keys.pendingProjectUpdates({ project_id: projectId }),
'project',
callback
)
},
/**
* Get the length of a project's per-project queue.
*
* @param {string} projectId
* @param {function(Error, number=): void} callback
*/
getProjectUpdatesLength(projectId, callback) {
rclient.llen(
Keys.pendingProjectUpdates({ project_id: projectId }),
callback
)
},
/**
* Publish a canary message on the project's editor-events channel with the
* sizes of the applied op's ack and broadcast payloads.
*
* @param {{projectId: string, docId: string, op: Object}} args
*/
sendCanaryAppliedOp({ projectId, docId, op }) {
const ack = JSON.stringify({ v: op.v, doc: docId }).length
// Updates with op.dup===true will not get sent to other clients, they only get acked.
@@ -110,6 +167,11 @@ const RealTimeRedisManager = {
}
},
/**
* Publish an applied-ops message, stamped with a unique message id.
*
* @param {Object} data
*/
sendData(data) {
// create a unique message id using a counter
const messageId = `doc:${HOST}:${RND}-${COUNT++}`
services/document-updater/app/js/UpdateKeys.js
+19 -1
View File
@@ -1,10 +1,28 @@
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
module.exports = {
/**
* Build a legacy `project_id:doc_id` dispatch marker.
*
* @param {string} projectId
* @param {string} docId
* @return {string}
*/
combineProjectIdAndDocId(projectId, docId) {
return `${projectId}:${docId}`
},
/**
* Split a dispatch marker into project id and doc id. A marker is either
* the legacy `project_id:doc_id` or, after the queue migration, a bare
* `project_id`. Project and doc ids never contain a colon, so splitting is
* unambiguous; for the bare form docId is undefined.
*
* @param {string} projectAndDocId
* @return {[string, string | undefined]}
*/
splitProjectIdAndDocId(projectAndDocId) {
return projectAndDocId.split(':')
const [projectId, docId] = projectAndDocId.split(':')
return [projectId, docId]
},
}
services/document-updater/app/js/UpdateManager.js
+276 -25
View File
@@ -1,5 +1,6 @@
// @ts-check
const Settings = require('@overleaf/settings')
const { callbackifyAll } = require('@overleaf/promise-utils')
const LockManager = require('./LockManager')
const ProjectLockManager = require('./ProjectLockManager')
@@ -24,6 +25,13 @@ const HistoryOTUpdateManager = require('./HistoryOTUpdateManager')
*/
const UpdateManager = {
/**
* Process the pending updates on a single doc's legacy per-doc queue. The
* caller must hold the doc lock.
*
* @param {string} projectId
* @param {string} docId
*/
async processOutstandingUpdates(projectId, docId) {
const timer = new Metrics.Timer('updateManager.processOutstandingUpdates')
try {
@@ -35,10 +43,17 @@ const UpdateManager = {
}
},
async processOutstandingUpdatesWithLock(projectId, docId) {
/**
* Process the pending updates for a project under the project lock, then
* keep going for as long as more updates are queued up.
*
* @param {string} projectId
* @param {string} [docIdHint] - doc id from a legacy dispatch marker, if any
*/
async processOutstandingUpdatesWithLock(projectId, docIdHint) {
const profile = new Profiler('processOutstandingUpdatesWithLock', {
project_id: projectId,
doc_id: docId,
doc_id: docIdHint,
})
// Always take the project lock before the per-doc lock to avoid deadlocks.
@@ -49,34 +64,247 @@ const UpdateManager = {
profile.log('getProjectLock')
try {
const lockValue = await LockManager.promises.tryLock(docId)
if (lockValue == null) {
return
}
profile.log('tryLock')
try {
await UpdateManager.processOutstandingUpdates(projectId, docId)
profile.log('processOutstandingUpdates')
} finally {
await LockManager.promises.releaseLock(docId, lockValue)
profile.log('releaseLock')
}
await UpdateManager.processOutstandingProjectUpdates(
projectId,
docIdHint,
projectLockValue,
profile
)
profile.log('processOutstandingProjectUpdates')
} finally {
await ProjectLockManager.promises.releaseLock(projectId, projectLockValue)
profile.log('releaseProjectLock').end()
}
await UpdateManager.continueProcessingUpdatesWithLock(projectId, docId)
await UpdateManager.continueProcessingUpdatesWithLock(projectId, docIdHint)
},
async continueProcessingUpdatesWithLock(projectId, docId) {
const length = await RealTimeRedisManager.promises.getUpdatesLength(docId)
if (length > 0) {
await UpdateManager.processOutstandingUpdatesWithLock(projectId, docId)
/**
* Process the pending updates for a project while holding the project lock.
*
* In phases 1 & 2 this drains the legacy per-doc queues (taking the per-doc
* lock for each doc) as well as the shared per-project queue; in phase 3
* only the per-project queue is processed.
*
* @param {string} projectId
* @param {string | undefined} docIdHint - doc id from a legacy dispatch
* marker, if any
* @param {string} projectLockValue - lock value of the held project lock
* @param {Profiler} profile - profile started by the caller
*/
async processOutstandingProjectUpdates(
projectId,
docIdHint,
projectLockValue,
profile
) {
switch (Settings.pendingUpdatesMigrationPhase) {
case 1:
case 2:
await UpdateManager.processOutstandingDocUpdates(
projectId,
docIdHint,
projectLockValue,
profile
)
// Phases 1 & 2 drain the per-project queue as well, as a safety net in
// case a producer writes it ahead of the phase switch.
// falls through
case 3:
await UpdateManager.fetchAndApplyProjectUpdates(projectId, profile)
break
default:
throw new Error(
`invalid pendingUpdatesMigrationPhase: ${Settings.pendingUpdatesMigrationPhase}`
)
}
},
/**
* Drain the legacy per-doc queues of all the docs in the project, plus the
* hinted doc, while holding the project lock.
*
* @param {string} projectId
* @param {string | undefined} docIdHint - doc id from a legacy dispatch
* marker, if any
* @param {string} projectLockValue - lock value of the held project lock
* @param {Profiler} profile - profile started by the caller
*/
async processOutstandingDocUpdates(
projectId,
docIdHint,
projectLockValue,
profile
) {
const docIds = new Set(
await RedisManager.promises.getDocIdsInProject(projectId)
)
profile.log('getDocIdsInProject')
if (docIdHint) {
docIds.add(docIdHint)
}
// Listing the docs above may have taken a while, so bump before we start
// processing and again after each doc. The final bump also covers draining
// the per-project queue afterwards.
await ProjectLockManager.promises.extendLock(projectId, projectLockValue)
profile.log('extendProjectLock')
for (const docId of docIds) {
await UpdateManager.fetchAndApplyDocUpdatesUnderDocLock(
projectId,
docId,
profile
)
await ProjectLockManager.promises.extendLock(projectId, projectLockValue)
profile.log('extendProjectLock')
}
},
/**
* Drain (one batch of) a single doc's legacy per-doc queue, taking the
* per-doc lock so that an old document-updater instance - which only takes
* the per-doc lock - cannot process the same doc concurrently during a
* rolling deploy. tryLock and skip on contention; the doc is revisited by
* the continuation check or a later dispatch.
*
* @param {string} projectId
* @param {string} docId
* @param {Profiler} profile - profile started by the caller
*/
async fetchAndApplyDocUpdatesUnderDocLock(projectId, docId, profile) {
const length = await RealTimeRedisManager.promises.getUpdatesLength(docId)
profile.log('getUpdatesLength')
if (length === 0) {
return
}
const lockValue = await LockManager.promises.tryLock(docId)
if (lockValue == null) {
return
}
profile.log('tryLock')
try {
await UpdateManager.processOutstandingUpdates(projectId, docId)
profile.log('processOutstandingUpdates')
} finally {
await LockManager.promises.releaseLock(docId, lockValue)
profile.log('releaseLock')
}
},
/**
* Apply one batch of updates from the shared per-project queue. We already
* hold the project lock, so no per-doc lock is required: the project lock
* serialises all processing for the project. Each update carries its `doc`
* id - real-time forces `update.doc = docId` before queueing it.
*
* @param {string} projectId
* @param {Profiler} profile - profile started by the caller
*/
async fetchAndApplyProjectUpdates(projectId, profile) {
const updates =
await RealTimeRedisManager.promises.getPendingProjectUpdates(projectId)
profile.log('getPendingProjectUpdates')
if (updates.length === 0) {
return
}
for (const update of updates) {
const docId = update.doc
if (HistoryOTUpdateManager.isHistoryOTEditOperationUpdate(update)) {
await HistoryOTUpdateManager.applyUpdate(projectId, docId, update)
} else {
await UpdateManager.applyUpdate(projectId, docId, update)
}
profile.log('applyProjectUpdate')
}
},
/**
* Total pending updates for a project: the per-project queue plus (in phases
* 1 & 2) the loaded docs' legacy per-doc queues. Used to decide whether to
* keep processing after releasing the lock - other workers' markers may have
* been consumed (and dropped on a failed tryLock) while we held it.
*
* @param {string} projectId
* @param {string} [docIdHint] - doc id from a legacy dispatch marker, if any
* @return {Promise<number>}
*/
async getProjectPendingUpdatesLength(projectId, docIdHint) {
switch (Settings.pendingUpdatesMigrationPhase) {
case 1:
case 2: {
const length =
await RealTimeRedisManager.promises.getProjectUpdatesLength(projectId)
if (length > 0) {
return length
}
return await UpdateManager.getDocPendingUpdatesLength(
projectId,
docIdHint
)
}
case 3:
return await RealTimeRedisManager.promises.getProjectUpdatesLength(
projectId
)
default:
throw new Error(
`invalid pendingUpdatesMigrationPhase: ${Settings.pendingUpdatesMigrationPhase}`
)
}
},
/**
* Pending updates across the legacy per-doc queues of the docs in the
* project, plus the hinted doc. Returns early with the first non-empty
* queue's length.
*
* @param {string} projectId
* @param {string} [docIdHint] - doc id from a legacy dispatch marker, if any
* @return {Promise<number>}
*/
async getDocPendingUpdatesLength(projectId, docIdHint) {
const docIds = new Set(
await RedisManager.promises.getDocIdsInProject(projectId)
)
if (docIdHint) {
docIds.add(docIdHint)
}
for (const docId of docIds) {
const length = await RealTimeRedisManager.promises.getUpdatesLength(docId)
if (length > 0) {
return length
}
}
return 0
},
/**
* Process the project's pending updates under the project lock if any are
* queued up.
*
* @param {string} projectId
* @param {string} [docIdHint] - doc id from a legacy dispatch marker, if any
*/
async continueProcessingUpdatesWithLock(projectId, docIdHint) {
const length = await UpdateManager.getProjectPendingUpdatesLength(
projectId,
docIdHint
)
if (length > 0) {
await UpdateManager.processOutstandingUpdatesWithLock(
projectId,
docIdHint
)
}
},
/**
* Apply one batch of updates from a doc's legacy per-doc queue. The caller
* must hold the doc lock.
*
* @param {string} projectId
* @param {string} docId
*/
async fetchAndApplyUpdates(projectId, docId) {
const profile = new Profiler('fetchAndApplyUpdates', {
project_id: projectId,
@@ -286,26 +514,43 @@ const UpdateManager = {
}
},
/**
* Process the project's pending updates, then run the given method on the
* doc, all under the project and doc locks.
*
* @param {Function} method - called with (projectId, docId, ...args)
* @param {string} projectId
* @param {string} docId
* @param {...any} args
* @return {Promise<any>} the return value of `method`
*/
async lockUpdatesAndDo(method, projectId, docId, ...args) {
const profile = new Profiler('lockUpdatesAndDo', {
project_id: projectId,
doc_id: docId,
})
// Always take the project lock before the per-doc lock to avoid deadlocks.
// Project lock first (always before the per-doc lock, to avoid deadlocks)
// so we can safely drain the shared per-project queue, then the per-doc
// lock for the operation itself.
const projectLockValue =
await ProjectLockManager.promises.getLock(projectId)
profile.log('getProjectLock')
let result
try {
await UpdateManager.processOutstandingProjectUpdates(
projectId,
docId,
projectLockValue,
profile
)
profile.log('processOutstandingProjectUpdates')
const lockValue = await LockManager.promises.getLock(docId)
profile.log('getLock')
try {
await UpdateManager.processOutstandingUpdates(projectId, docId)
profile.log('processOutstandingUpdates')
result = await method(projectId, docId, ...args)
profile.log('method')
} finally {
@@ -334,6 +579,12 @@ const UpdateManager = {
return result
},
/**
* Replace unpaired surrogate characters in the update's inserts.
*
* @param {Update} update
* @return {Update}
*/
_sanitizeUpdate(update) {
// In Javascript, characters are 16-bits wide. It does not understand surrogates as characters.
//
@@ -347,7 +598,7 @@ const UpdateManager = {
// Something must be going on client side that is screwing up the encoding and splitting the
// two 16-bit characters so that \uD835 is standalone.
for (const op of update.op || []) {
if (op.i != null) {
if (isInsert(op)) {
// Replace high and low surrogate characters with 'replacement character' (\uFFFD)
op.i = op.i.replace(/[\uD800-\uDFFF]/g, '\uFFFD')
}
services/document-updater/config/settings.defaults.js
+14
View File
@@ -143,6 +143,9 @@ module.exports = {
pendingUpdates({ doc_id: docId }) {
return `PendingUpdates:{${docId}}`
},
pendingProjectUpdates({ project_id: projectId }) {
return `PendingProjectUpdates:{${projectId}}`
},
lastUpdatedBy({ doc_id: docId }) {
return `lastUpdatedBy:{${docId}}`
},
@@ -171,6 +174,17 @@ module.exports = {
dispatcherCount: parseInt(process.env.DISPATCHER_COUNT || 10, 10),
// Migration of the real-time -> document-updater update queue from a per-doc
// queue to a per-project queue. See
// services/document-updater/scripts/check_migration_phase_switch.js
// phase 1: legacy per-doc queue only
// phase 2: producer writes the per-project queue; consumer drains both
// phase 3: per-project queue only
pendingUpdatesMigrationPhase: parseInt(
process.env.PENDING_UPDATES_MIGRATION_PHASE || '1',
10
),
redisLockTTLSeconds: 30,
mongo: {
services/document-updater/scripts/check_migration_phase_switch.js
+214
View File
@@ -0,0 +1,214 @@
/**
* Pre-flight check for switching the real-time -> document-updater update queue
* migration phase (PENDING_UPDATES_MIGRATION_PHASE).
*
* phase 1: legacy per-doc queue only
* phase 2: producer writes the per-project queue; consumer drains both
* phase 3: per-project queue only
*
* Going to phase 3 is only safe once no data can be stranded in the legacy
* per-doc queues (because the phase-3 consumer stops reading them). This script
* verifies that and exits non-zero when the switch is unsafe.
*
* It is READ-ONLY (SCAN / LLEN / LRANGE only - never KEYS) and safe to run at
* any time. In a redis cluster it SCANs every master node, since the per-doc
* keys are spread across slots.
*
* Usage:
* node scripts/check_migration_phase_switch.js --current 2 --target 3
*/
const { setTimeout } = require('node:timers/promises')
const minimist = require('minimist')
const Settings = require('@overleaf/settings')
const { rclient } = require('./../app/js/RedisManager')
// Window size when walking a dispatch list tail -> head.
const LIST_CHUNK_SIZE = 1000
/**
* List all cluster master nodes, or the plain client outside a cluster.
*
* @return {Array<Object>}
*/
function getNodes() {
return (
(typeof rclient.nodes === 'function'
? rclient.nodes('master')
: undefined) || [rclient]
)
}
/**
* Non-blocking SCAN across all cluster master nodes.
*
* @param {string} pattern - a redis MATCH pattern
* @yields {string} matching keys
*/
async function* scanKeys(pattern) {
for (const node of getNodes()) {
let cursor = '0'
do {
const [next, keys] = await node.scan(
cursor,
'MATCH',
pattern,
'COUNT',
1000
)
cursor = next
for (const key of keys) {
yield key
}
if (cursor !== '0') {
await setTimeout(10) // be gentle on redis
}
} while (cursor !== '0')
}
}
/**
* Are there any non-empty legacy per-doc queues left?
*
* @return {Promise<{nonEmptyDocs: number, totalOps: number}>}
*/
async function checkPerDocQueues() {
let nonEmptyDocs = 0
let totalOps = 0
for await (const key of scanKeys('PendingUpdates:{*}')) {
const length = await rclient.llen(key)
if (length > 0) {
nonEmptyDocs++
totalOps += length
}
}
return { nonEmptyDocs, totalOps }
}
/**
* Are there any legacy "<proj>:<doc>" markers still sitting on the dispatch
* lists? A colon means a phase-1 producer is still writing. We snapshot the
* length and walk each list tail -> head in bounded windows, rather than
* materialising the whole list in one LRANGE. The lists are only popped from
* the head (BLPOP) and appended at the tail (RPUSH), so an entry's index only
* ever decreases; sweeping tail -> head with contiguous windows therefore
* never skips an entry that is still queued (though a fast-draining entry can
* ride the window boundary down and be read several times - harmless for a
* colon scan). The only entries we don't observe are ones drained mid-walk
* (already processed) or pushed after we started. The exhaustive per-doc SCAN
* above is in any case the authoritative gate; a colon found here is just a
* strong extra signal.
*
* @return {Promise<{legacyMarkers: number, inspectedEntries: number}>}
*/
async function checkDispatchListsForLegacyMarkers() {
const shardCount = Settings.dispatcherCount
let legacyMarkers = 0
let inspectedEntries = 0
for (let shard = 0; shard < shardCount; shard++) {
const listKey =
shard === 0 ? 'pending-updates-list' : `pending-updates-list-${shard}`
const length = await rclient.llen(listKey)
for (let end = length; end > 0; end -= LIST_CHUNK_SIZE) {
const start = Math.max(0, end - LIST_CHUNK_SIZE)
const entries = await rclient.lrange(listKey, start, end - 1)
for (const marker of entries) {
inspectedEntries++
if (marker.includes(':')) {
legacyMarkers++
}
}
}
}
return { legacyMarkers, inspectedEntries }
}
/**
* Check whether switching the migration phase is safe.
*
* @param {number} current - the phase currently running
* @param {number} target - the phase to switch to
* @return {Promise<boolean>} whether the switch is safe
*/
async function checkSwitch(current, target) {
console.log(`Checking switch from phase ${current} -> ${target}`)
if (Math.abs(target - current) > 1) {
console.warn(
`WARNING: ${current} -> ${target} is not an adjacent transition. ` +
'Roll the phase one step at a time.'
)
}
// The only gated transition is enabling phase 3 (consumer stops reading the
// per-doc queues). Everything else is safe with the dual-read consumer.
if (target < 3) {
console.log(
'Target phase < 3: the dual-read consumer drains both queues, so this ' +
'switch (incl. roll-backs) is safe.'
)
return true
}
const { nonEmptyDocs, totalOps } = await checkPerDocQueues()
const { legacyMarkers, inspectedEntries } =
await checkDispatchListsForLegacyMarkers()
console.log(
`Legacy per-doc queues with pending updates: ${nonEmptyDocs} (${totalOps} ops)`
)
console.log(
`Legacy "<proj>:<doc>" markers found across ${inspectedEntries} dispatch ` +
`list entries: ${legacyMarkers}`
)
const safe = nonEmptyDocs === 0 && legacyMarkers === 0
if (safe) {
console.log(
'SAFE: no legacy per-doc data remains; phase 3 will not strand updates.'
)
} else {
console.error(
'UNSAFE: legacy per-doc data still present. Keep the producer at phase ' +
'>= 2 and wait for the per-doc queues to drain before switching to ' +
'phase 3.'
)
}
return safe
}
/**
* Parse the command line arguments and run the check.
*
* @return {Promise<boolean>} whether the switch is safe
*/
async function main() {
const argv = minimist(process.argv.slice(2), {
boolean: ['help'],
alias: { h: 'help' },
})
if (argv.help || argv.current == null || argv.target == null) {
console.log(`
Usage: node scripts/check_migration_phase_switch.js --current N --target M
Options:
--current The phase currently running (1, 2 or 3)
--target The phase you want to switch to (1, 2 or 3)
--help, -h Show this help message
`)
process.exit(argv.help ? 0 : 1)
}
const current = parseInt(argv.current, 10)
const target = parseInt(argv.target, 10)
return await checkSwitch(current, target)
}
main()
.then(safe => {
process.exit(safe ? 0 : 1)
})
.catch(error => {
console.error('Error running phase-switch check', error)
process.exit(2)
})
services/document-updater/test/acceptance/js/QueueMigrationTests.js
+181
View File
@@ -0,0 +1,181 @@
/*
* Exercises the migration of the real-time -> document-updater update queue
* from the per-doc queue to the per-project queue.
*
* The consumer's behaviour is gated by Settings.pendingUpdatesMigrationPhase,
* which the app reads live. The acceptance app runs in-process, so each block
* sets the phase it needs in a `before` hook and the top-level `after` restores
* the original value.
*
* The dual-read consumer (phases 1 & 2) must drain BOTH the legacy per-doc queue
* and the new per-project queue, so updates queued either side of a producer
* roll-forward/roll-backward are still applied. Ordering is relaxed (ShareJS
* rebases by version); these tests keep versions monotonic across the switch.
*/
const { expect } = require('chai')
const { setTimeout } = require('node:timers/promises')
const Settings = require('@overleaf/settings')
const rclientDU = require('@overleaf/redis-wrapper').createClient(
Settings.redis.documentupdater
)
const Keys = Settings.redis.documentupdater.key_schema
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
function insertOp(text, position, version, docId) {
return { doc: docId, op: [{ i: text, p: position }], v: version }
}
describe('Queue migration', function () {
before(function () {
this.originalPhase = Settings.pendingUpdatesMigrationPhase
})
after(function () {
Settings.pendingUpdatesMigrationPhase = this.originalPhase
})
beforeEach(async function () {
this.lines = ['one', 'two', 'three']
this.version = 0
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
})
await DocUpdaterApp.ensureRunning()
await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
})
describe('dual-read consumer (phases 1 & 2)', function () {
before(function () {
Settings.pendingUpdatesMigrationPhase = 2
})
it('roll-forward: legacy updates then per-project updates all apply in order', async function () {
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
insertOp('AAA\n', 0, 0, this.doc_id),
{ toProjectQueue: false }
)
await DocUpdaterClient.waitForPendingUpdates(this.project_id, this.doc_id)
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
insertOp('BBB\n', 0, 1, this.doc_id),
{ toProjectQueue: true }
)
await DocUpdaterClient.waitForPendingUpdates(this.project_id, this.doc_id)
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.lines.should.deep.equal(['BBB', 'AAA', 'one', 'two', 'three'])
doc.version.should.equal(2)
})
it('roll-backward: per-project updates then legacy updates all apply in order', async function () {
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
insertOp('AAA\n', 0, 0, this.doc_id),
{ toProjectQueue: true }
)
await DocUpdaterClient.waitForPendingUpdates(this.project_id, this.doc_id)
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
insertOp('BBB\n', 0, 1, this.doc_id),
{ toProjectQueue: false }
)
await DocUpdaterClient.waitForPendingUpdates(this.project_id, this.doc_id)
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.lines.should.deep.equal(['BBB', 'AAA', 'one', 'two', 'three'])
doc.version.should.equal(2)
})
it('straddle: updates present in BOTH queues drain together (per-doc first), in order', async function () {
// Populate both queues before any dispatch, then trigger a single
// whole-project drain with a bare project-id marker.
await rclientDU.rpush(
Keys.pendingUpdates({ doc_id: this.doc_id }),
JSON.stringify(insertOp('AAA\n', 0, 0, this.doc_id))
)
await rclientDU.rpush(
Keys.pendingProjectUpdates({ project_id: this.project_id }),
JSON.stringify(insertOp('BBB\n', 0, 1, this.doc_id))
)
await rclientDU.rpush('pending-updates-list', this.project_id)
await DocUpdaterClient.waitForPendingUpdates(this.project_id, this.doc_id)
await setTimeout(200)
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.lines.should.deep.equal(['BBB', 'AAA', 'one', 'two', 'three'])
doc.version.should.equal(2)
})
it('flush applies per-project queue updates first (no stale flush)', async function () {
// Queue an update on the per-project queue WITHOUT a dispatch marker, so
// it is only picked up by the flush path draining outstanding updates.
await rclientDU.rpush(
Keys.pendingProjectUpdates({ project_id: this.project_id }),
JSON.stringify(insertOp('AAA\n', 0, 0, this.doc_id))
)
await DocUpdaterClient.flushDoc(this.project_id, this.doc_id)
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.lines.should.deep.equal(['AAA', 'one', 'two', 'three'])
doc.version.should.equal(1)
const remaining = await rclientDU.llen(
Keys.pendingProjectUpdates({ project_id: this.project_id })
)
expect(remaining).to.equal(0)
})
})
describe('project-queue-only consumer (phase 3)', function () {
before(function () {
Settings.pendingUpdatesMigrationPhase = 3
})
it('drains the per-project queue', async function () {
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
insertOp('AAA\n', 0, 0, this.doc_id),
{ toProjectQueue: true }
)
await DocUpdaterClient.waitForPendingUpdates(this.project_id, this.doc_id)
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.lines.should.deep.equal(['AAA', 'one', 'two', 'three'])
doc.version.should.equal(1)
})
it('ignores the legacy per-doc queue', async function () {
// Push to the legacy per-doc queue with its marker; the phase-3 consumer
// must NOT process it (it no longer reads the per-doc queue).
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
insertOp('AAA\n', 0, 0, this.doc_id),
{ toProjectQueue: false }
)
await setTimeout(500)
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.lines.should.deep.equal(this.lines)
doc.version.should.equal(0)
const remaining = await rclientDU.llen(
Keys.pendingUpdates({ doc_id: this.doc_id })
)
expect(remaining).to.equal(1)
})
})
})
services/document-updater/test/acceptance/js/SettingADocumentTests.js
+1 -1
View File
@@ -546,7 +546,7 @@ describe('Setting a document', function () {
this.doc_id,
this.historyOTUpdate
)
await DocUpdaterClient.waitForPendingUpdates(this.doc_id)
await DocUpdaterClient.waitForPendingUpdates(this.project_id, this.doc_id)
})
afterEach(function () {
services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js
+34 -14
View File
@@ -36,32 +36,52 @@ module.exports = DocUpdaterClient = {
rclientSub.on('message', messageHandler)
},
async sendUpdate(projectId, docId, update) {
const docKey = `${projectId}:${docId}`
await rclient.rpush(
keys.pendingUpdates({ doc_id: docId }),
JSON.stringify(update)
)
await rclient.sadd('DocsWithPendingUpdates', docKey)
await rclient.rpush(getPendingUpdateListKey(), docKey)
// Enqueue an update for the doc-updater. By default the queue format follows
// the configured migration phase, but tests can force a format with
// options.toProjectQueue to exercise the migration explicitly:
// phase 1 / toProjectQueue=false: legacy per-doc queue + "<proj>:<doc>" marker
// phase >= 2 / toProjectQueue=true: per-project queue + bare "<proj>" marker
async sendUpdate(projectId, docId, update, options = {}) {
const toProjectQueue =
options.toProjectQueue ?? Settings.pendingUpdatesMigrationPhase >= 2
if (toProjectQueue) {
await rclient.rpush(
keys.pendingProjectUpdates({ project_id: projectId }),
JSON.stringify({ ...update, doc: docId })
)
await rclient.rpush(getPendingUpdateListKey(), projectId)
} else {
await rclient.rpush(
keys.pendingUpdates({ doc_id: docId }),
JSON.stringify(update)
)
await rclient.rpush(getPendingUpdateListKey(), `${projectId}:${docId}`)
}
},
async sendUpdates(projectId, docId, updates) {
async sendUpdates(projectId, docId, updates, options = {}) {
await DocUpdaterClient.preloadDoc(projectId, docId)
for (const update of updates) {
await DocUpdaterClient.sendUpdate(projectId, docId, update)
await DocUpdaterClient.sendUpdate(projectId, docId, update, options)
}
await DocUpdaterClient.waitForPendingUpdates(docId)
await DocUpdaterClient.waitForPendingUpdates(projectId, docId)
},
async waitForPendingUpdates(docId) {
// Wait until both the legacy per-doc queue and the per-project queue are
// drained, so the helper works regardless of the migration phase in use.
async waitForPendingUpdates(projectId, docId) {
const maxRetries = 30
const retryInterval = 100
for (let attempt = 0; attempt < maxRetries; attempt++) {
const length = await rclient.llen(keys.pendingUpdates({ doc_id: docId }))
const docLength = await rclient.llen(
keys.pendingUpdates({ doc_id: docId })
)
const projectLength = await rclient.llen(
keys.pendingProjectUpdates({ project_id: projectId })
)
if (length === 0) {
if (docLength === 0 && projectLength === 0) {
return // Success - no pending updates
}
services/document-updater/test/unit/js/DispatchManager/DispatchManagerTests.js
+22
View File
@@ -139,6 +139,28 @@ describe('DispatchManager', function () {
})
})
describe('with a bare project-id marker (per-project queue)', function () {
beforeEach(function () {
this.client.blpop = sinon
.stub()
.callsArgWith(2, null, ['pending-updates-list', this.project_id])
this.UpdateManager.processOutstandingUpdatesWithLock = sinon
.stub()
.callsArg(2)
return this.worker._waitForUpdateThenDispatchWorker(this.callback)
})
it('should call processOutstandingUpdatesWithLock with no doc hint', function () {
this.UpdateManager.processOutstandingUpdatesWithLock
.calledWith(this.project_id, undefined)
.should.equal(true)
})
it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
})
describe('pending updates list with shard key', function () {
beforeEach(function (done) {
this.client = {
services/document-updater/test/unit/js/RealTimeRedisManager/RealTimeRedisManagerTests.js
+82
View File
@@ -35,6 +35,9 @@ describe('RealTimeRedisManager', function () {
pendingUpdates({ doc_id: docId }) {
return `PendingUpdates:${docId}`
},
pendingProjectUpdates({ project_id: projectId }) {
return `PendingProjectUpdates:${projectId}`
},
},
}),
pubsub: {
@@ -140,6 +143,85 @@ describe('RealTimeRedisManager', function () {
})
})
describe('getPendingProjectUpdates', function () {
beforeEach(function () {
this.rclient.llen = sinon.stub()
this.rclient.lrange = sinon.stub()
this.rclient.ltrim = sinon.stub()
})
describe('successfully', function () {
beforeEach(function () {
this.updates = [
{ doc: 'doc-1', op: [{ i: 'foo', p: 4 }] },
{ doc: 'doc-2', op: [{ i: 'bar', p: 6 }] },
]
this.jsonUpdates = this.updates.map(update => JSON.stringify(update))
this.rclient.exec = sinon.stub().yields(null, [2, this.jsonUpdates])
return this.RealTimeRedisManager.getPendingProjectUpdates(
this.project_id,
this.callback
)
})
it('should get the pending updates from the per-project queue', function () {
return this.rclient.lrange
.calledWith(`PendingProjectUpdates:${this.project_id}`, 0, 7)
.should.equal(true)
})
it('should delete the pending updates', function () {
return this.rclient.ltrim
.calledWith(`PendingProjectUpdates:${this.project_id}`, 8, -1)
.should.equal(true)
})
return it('should call the callback with the updates', function () {
return this.callback.calledWith(null, this.updates).should.equal(true)
})
})
return describe("when the JSON doesn't parse", function () {
beforeEach(function () {
this.jsonUpdates = [
JSON.stringify({ doc: 'doc-1', op: [{ i: 'foo', p: 4 }] }),
'broken json',
]
this.rclient.exec = sinon.stub().yields(null, [2, this.jsonUpdates])
return this.RealTimeRedisManager.getPendingProjectUpdates(
this.project_id,
this.callback
)
})
return it('should return an error to the callback', function () {
return this.callback
.calledWith(sinon.match.has('name', 'SyntaxError'))
.should.equal(true)
})
})
})
describe('getProjectUpdatesLength', function () {
beforeEach(function () {
this.rclient.llen = sinon.stub().yields(null, (this.length = 4))
return this.RealTimeRedisManager.getProjectUpdatesLength(
this.project_id,
this.callback
)
})
it('should look up the length of the per-project queue', function () {
return this.rclient.llen
.calledWith(`PendingProjectUpdates:${this.project_id}`)
.should.equal(true)
})
return it('should return the length', function () {
return this.callback.calledWith(null, this.length).should.equal(true)
})
})
return describe('sendData', function () {
beforeEach(function () {
this.message_id = 'doc:somehost:01020304-0'
services/document-updater/test/unit/js/UpdateManager/UpdateManagerTests.js
+313 -59
View File
@@ -36,6 +36,7 @@ describe('UpdateManager', function () {
promises: {
tryLock: sinon.stub().resolves(this.projectLockValue),
getLock: sinon.stub().resolves(this.projectLockValue),
extendLock: sinon.stub().resolves(),
releaseLock: sinon.stub().resolves(),
},
}
@@ -45,17 +46,25 @@ describe('UpdateManager', function () {
setDocument: sinon.stub().resolves(),
updateDocument: sinon.stub(),
recordProjectNotificationTimestamp: sinon.stub().resolves(),
getDocIdsInProject: sinon.stub().resolves([]),
},
}
this.RealTimeRedisManager = {
sendData: sinon.stub(),
promises: {
getUpdatesLength: sinon.stub(),
getUpdatesLength: sinon.stub().resolves(0),
getPendingUpdatesForDoc: sinon.stub(),
getProjectUpdatesLength: sinon.stub().resolves(0),
getPendingProjectUpdates: sinon.stub().resolves([]),
},
}
this.HistoryOTUpdateManager = {
isHistoryOTEditOperationUpdate: sinon.stub().returns(false),
applyUpdate: sinon.stub().resolves(),
}
this.ShareJsUpdateManager = {
promises: {
applyUpdate: sinon.stub(),
@@ -66,7 +75,7 @@ describe('UpdateManager', function () {
recordAndFlushHistoryOps: sinon.stub(),
}
this.Settings = {}
this.Settings = { pendingUpdatesMigrationPhase: 1 }
this.DocumentManager = {
promises: {
@@ -106,6 +115,7 @@ describe('UpdateManager', function () {
'./RealTimeRedisManager': this.RealTimeRedisManager,
'./ShareJsUpdateManager': this.ShareJsUpdateManager,
'./HistoryManager': this.HistoryManager,
'./HistoryOTUpdateManager': this.HistoryOTUpdateManager,
'./Metrics': this.Metrics,
'@overleaf/settings': this.Settings,
'./DocumentManager': this.DocumentManager,
@@ -139,12 +149,12 @@ describe('UpdateManager', function () {
})
describe('processOutstandingUpdatesWithLock', function () {
describe('when the lock is free', function () {
describe('when the project lock is free', function () {
beforeEach(function () {
this.UpdateManager.promises.continueProcessingUpdatesWithLock = sinon
.stub()
.resolves()
this.UpdateManager.promises.processOutstandingUpdates = sinon
this.UpdateManager.promises.processOutstandingProjectUpdates = sinon
.stub()
.resolves()
})
@@ -157,25 +167,10 @@ describe('UpdateManager', function () {
)
})
it('should acquire the project lock before the doc lock', function () {
it('should acquire the project lock', function () {
this.ProjectLockManager.promises.getLock
.calledWith(this.project_id)
.should.equal(true)
this.LockManager.promises.tryLock
.calledAfter(this.ProjectLockManager.promises.getLock)
.should.equal(true)
})
it('should acquire the lock', function () {
this.LockManager.promises.tryLock
.calledWith(this.doc_id)
.should.equal(true)
})
it('should free the lock', function () {
this.LockManager.promises.releaseLock
.calledWith(this.doc_id, this.lockValue)
.should.equal(true)
})
it('should free the project lock', function () {
@@ -184,18 +179,18 @@ describe('UpdateManager', function () {
.should.equal(true)
})
it('should process the outstanding updates', function () {
this.UpdateManager.promises.processOutstandingUpdates
.calledWith(this.project_id, this.doc_id)
it('should process the outstanding project updates', function () {
this.UpdateManager.promises.processOutstandingProjectUpdates
.calledWith(this.project_id, this.doc_id, this.projectLockValue)
.should.equal(true)
})
it('should do everything with the lock acquired', function () {
this.UpdateManager.promises.processOutstandingUpdates
.calledAfter(this.LockManager.promises.tryLock)
this.UpdateManager.promises.processOutstandingProjectUpdates
.calledAfter(this.ProjectLockManager.promises.getLock)
.should.equal(true)
this.UpdateManager.promises.processOutstandingUpdates
.calledBefore(this.LockManager.promises.releaseLock)
this.UpdateManager.promises.processOutstandingProjectUpdates
.calledBefore(this.ProjectLockManager.promises.releaseLock)
.should.equal(true)
})
@@ -206,10 +201,10 @@ describe('UpdateManager', function () {
})
})
describe('when processOutstandingUpdates returns an error', function () {
describe('when processOutstandingProjectUpdates returns an error', function () {
beforeEach(async function () {
this.error = new Error('Something went wrong')
this.UpdateManager.promises.processOutstandingUpdates = sinon
this.UpdateManager.promises.processOutstandingProjectUpdates = sinon
.stub()
.rejects(this.error)
await expect(
@@ -220,12 +215,6 @@ describe('UpdateManager', function () {
).to.be.rejectedWith(this.error)
})
it('should free the lock', function () {
this.LockManager.promises.releaseLock
.calledWith(this.doc_id, this.lockValue)
.should.equal(true)
})
it('should free the project lock', function () {
this.ProjectLockManager.promises.releaseLock
.calledWith(this.project_id, this.projectLockValue)
@@ -233,37 +222,247 @@ describe('UpdateManager', function () {
})
})
})
})
describe('when the lock is taken', function () {
describe('processOutstandingProjectUpdates', function () {
beforeEach(function () {
this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock = sinon
.stub()
.resolves()
this.UpdateManager.promises.fetchAndApplyProjectUpdates = sinon
.stub()
.resolves()
})
describe('in phases 1 & 2', function () {
beforeEach(async function () {
this.LockManager.promises.tryLock.resolves(null)
this.UpdateManager.promises.processOutstandingUpdates = sinon
.stub()
.resolves()
await this.UpdateManager.promises.processOutstandingUpdatesWithLock(
this.Settings.pendingUpdatesMigrationPhase = 2
this.RedisManager.promises.getDocIdsInProject.resolves([
'doc-1',
'doc-2',
])
await this.UpdateManager.promises.processOutstandingProjectUpdates(
this.project_id,
this.doc_id,
this.projectLockValue,
new this.Profiler()
)
})
it('should drain the per-doc queue for every doc in the project plus the hint', function () {
this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock.should.have.been.calledWith(
this.project_id,
'doc-1'
)
this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock.should.have.been.calledWith(
this.project_id,
'doc-2'
)
this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock.should.have.been.calledWith(
this.project_id,
this.doc_id
)
})
it('should not process the updates', function () {
this.UpdateManager.promises.processOutstandingUpdates.called.should.equal(
it('should extend the project lock between docs', function () {
this.ProjectLockManager.promises.extendLock.should.have.been.calledWith(
this.project_id,
this.projectLockValue
)
})
it('should drain the per-project queue', function () {
this.UpdateManager.promises.fetchAndApplyProjectUpdates.should.have.been.calledWith(
this.project_id
)
})
})
describe('in phase 3', function () {
beforeEach(async function () {
this.Settings.pendingUpdatesMigrationPhase = 3
await this.UpdateManager.promises.processOutstandingProjectUpdates(
this.project_id,
this.doc_id,
this.projectLockValue,
new this.Profiler()
)
})
it('should not look up the docs in the project', function () {
this.RedisManager.promises.getDocIdsInProject.called.should.equal(false)
})
it('should not drain any per-doc queue', function () {
this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock.called.should.equal(
false
)
})
it('should free the project lock', function () {
this.ProjectLockManager.promises.releaseLock
.calledWith(this.project_id, this.projectLockValue)
it('should not bump the project lock (freshly acquired, no per-doc loop)', function () {
this.ProjectLockManager.promises.extendLock.called.should.equal(false)
})
it('should drain the per-project queue', function () {
this.UpdateManager.promises.fetchAndApplyProjectUpdates.should.have.been.calledWith(
this.project_id
)
})
})
})
describe('fetchAndApplyDocUpdatesUnderDocLock', function () {
beforeEach(function () {
this.UpdateManager.promises.processOutstandingUpdates = sinon
.stub()
.resolves()
})
describe('when the doc has pending updates', function () {
beforeEach(async function () {
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(2)
await this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock(
this.project_id,
this.doc_id,
new this.Profiler()
)
})
it('should take the per-doc lock', function () {
this.LockManager.promises.tryLock
.calledWith(this.doc_id)
.should.equal(true)
})
it('should process the doc updates and release the per-doc lock', function () {
this.UpdateManager.promises.processOutstandingUpdates.should.have.been.calledWith(
this.project_id,
this.doc_id
)
this.LockManager.promises.releaseLock
.calledWith(this.doc_id, this.lockValue)
.should.equal(true)
})
})
describe('when the doc has no pending updates', function () {
beforeEach(async function () {
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(0)
await this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock(
this.project_id,
this.doc_id,
new this.Profiler()
)
})
it('should not take the per-doc lock', function () {
this.LockManager.promises.tryLock.called.should.equal(false)
})
})
describe('when the per-doc lock is taken (e.g. by an old instance)', function () {
beforeEach(async function () {
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(2)
this.LockManager.promises.tryLock.resolves(null)
await this.UpdateManager.promises.fetchAndApplyDocUpdatesUnderDocLock(
this.project_id,
this.doc_id,
new this.Profiler()
)
})
it('should skip processing the doc', function () {
this.UpdateManager.promises.processOutstandingUpdates.called.should.equal(
false
)
})
})
})
describe('fetchAndApplyProjectUpdates', function () {
beforeEach(function () {
this.UpdateManager.promises.applyUpdate = sinon.stub().resolves()
})
describe('with updates carrying their doc id', function () {
beforeEach(async function () {
this.updates = [
{ doc: 'doc-1', op: 'a' },
{ doc: 'doc-2', op: 'b' },
]
this.RealTimeRedisManager.promises.getPendingProjectUpdates.resolves(
this.updates
)
await this.UpdateManager.promises.fetchAndApplyProjectUpdates(
this.project_id,
new this.Profiler()
)
})
it('should read the per-project queue', function () {
this.RealTimeRedisManager.promises.getPendingProjectUpdates
.calledWith(this.project_id)
.should.equal(true)
})
it('should apply each update to the doc it carries', function () {
this.UpdateManager.promises.applyUpdate.should.have.been.calledWith(
this.project_id,
'doc-1',
this.updates[0]
)
this.UpdateManager.promises.applyUpdate.should.have.been.calledWith(
this.project_id,
'doc-2',
this.updates[1]
)
})
})
describe('with a history-OT update', function () {
beforeEach(async function () {
this.update = { doc: 'doc-1', op: 'a' }
this.RealTimeRedisManager.promises.getPendingProjectUpdates.resolves([
this.update,
])
this.HistoryOTUpdateManager.isHistoryOTEditOperationUpdate.returns(true)
await this.UpdateManager.promises.fetchAndApplyProjectUpdates(
this.project_id,
new this.Profiler()
)
})
it('should route it to the history-OT update manager', function () {
this.HistoryOTUpdateManager.applyUpdate.should.have.been.calledWith(
this.project_id,
'doc-1',
this.update
)
this.UpdateManager.promises.applyUpdate.called.should.equal(false)
})
})
describe('with no updates', function () {
beforeEach(async function () {
this.RealTimeRedisManager.promises.getPendingProjectUpdates.resolves([])
await this.UpdateManager.promises.fetchAndApplyProjectUpdates(
this.project_id,
new this.Profiler()
)
})
it('should not apply any update', function () {
this.UpdateManager.promises.applyUpdate.called.should.equal(false)
})
})
})
describe('continueProcessingUpdatesWithLock', function () {
describe('when there are outstanding updates', function () {
beforeEach(async function () {
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(3)
this.UpdateManager.promises.getProjectPendingUpdatesLength = sinon
.stub()
.resolves(3)
this.UpdateManager.promises.processOutstandingUpdatesWithLock = sinon
.stub()
.resolves()
@@ -282,7 +481,9 @@ describe('UpdateManager', function () {
describe('when there are no outstanding updates', function () {
beforeEach(async function () {
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(0)
this.UpdateManager.promises.getProjectPendingUpdatesLength = sinon
.stub()
.resolves(0)
this.UpdateManager.promises.processOutstandingUpdatesWithLock = sinon
.stub()
.resolves()
@@ -300,6 +501,61 @@ describe('UpdateManager', function () {
})
})
describe('getProjectPendingUpdatesLength', function () {
describe('when the per-project queue is non-empty', function () {
beforeEach(async function () {
this.RealTimeRedisManager.promises.getProjectUpdatesLength.resolves(4)
this.length =
await this.UpdateManager.promises.getProjectPendingUpdatesLength(
this.project_id,
this.doc_id
)
})
it('should short-circuit without scanning per-doc queues', function () {
this.length.should.equal(4)
this.RedisManager.promises.getDocIdsInProject.called.should.equal(false)
})
})
describe('when the per-project queue is empty (phases 1 & 2)', function () {
beforeEach(async function () {
this.Settings.pendingUpdatesMigrationPhase = 2
this.RealTimeRedisManager.promises.getProjectUpdatesLength.resolves(0)
this.RedisManager.promises.getDocIdsInProject.resolves(['doc-1'])
this.RealTimeRedisManager.promises.getUpdatesLength
.withArgs('doc-1')
.resolves(2)
this.length =
await this.UpdateManager.promises.getProjectPendingUpdatesLength(
this.project_id,
this.doc_id
)
})
it('should fall back to the per-doc queues', function () {
this.length.should.equal(2)
})
})
describe('in phase 3', function () {
beforeEach(async function () {
this.Settings.pendingUpdatesMigrationPhase = 3
this.RealTimeRedisManager.promises.getProjectUpdatesLength.resolves(0)
this.length =
await this.UpdateManager.promises.getProjectPendingUpdatesLength(
this.project_id,
this.doc_id
)
})
it('should only consider the per-project queue', function () {
this.length.should.equal(0)
this.RedisManager.promises.getDocIdsInProject.called.should.equal(false)
})
})
})
describe('fetchAndApplyUpdates', function () {
describe('with updates', function () {
beforeEach(async function () {
@@ -902,7 +1158,7 @@ describe('UpdateManager', function () {
this.UpdateManager.promises.continueProcessingUpdatesWithLock = sinon
.stub()
.resolves()
this.UpdateManager.promises.processOutstandingUpdates = sinon
this.UpdateManager.promises.processOutstandingProjectUpdates = sinon
.stub()
.resolves()
this.response = await this.UpdateManager.promises.lockUpdatesAndDo(
@@ -925,10 +1181,11 @@ describe('UpdateManager', function () {
.should.equal(true)
})
it('should process any outstanding updates', function () {
this.UpdateManager.promises.processOutstandingUpdates.should.have.been.calledWith(
it('should process any outstanding updates for the project', function () {
this.UpdateManager.promises.processOutstandingProjectUpdates.should.have.been.calledWith(
this.project_id,
this.doc_id
this.doc_id,
this.projectLockValue
)
})
@@ -958,10 +1215,10 @@ describe('UpdateManager', function () {
})
})
describe('when processOutstandingUpdates returns an error', function () {
describe('when processOutstandingProjectUpdates returns an error', function () {
beforeEach(async function () {
this.error = new Error('Something went wrong')
this.UpdateManager.promises.processOutstandingUpdates = sinon
this.UpdateManager.promises.processOutstandingProjectUpdates = sinon
.stub()
.rejects(this.error)
await expect(
@@ -974,10 +1231,7 @@ describe('UpdateManager', function () {
).to.be.rejectedWith(this.error)
})
it('should free both locks', function () {
this.LockManager.promises.releaseLock
.calledWith(this.doc_id, this.lockValue)
.should.equal(true)
it('should free the project lock', function () {
this.ProjectLockManager.promises.releaseLock
.calledWith(this.project_id, this.projectLockValue)
.should.equal(true)
@@ -987,7 +1241,7 @@ describe('UpdateManager', function () {
describe('when the method returns an error', function () {
beforeEach(async function () {
this.error = new Error('something went wrong')
this.UpdateManager.promises.processOutstandingUpdates = sinon
this.UpdateManager.promises.processOutstandingProjectUpdates = sinon
.stub()
.resolves()
this.method = sinon.stub().rejects(this.error)
services/real-time/app/js/DocumentUpdaterManager.js
+113 -2
View File
@@ -22,6 +22,15 @@ const {
const rclient = RedisWrapper.createClient(settings.redis.documentupdater)
const Keys = settings.redis.documentupdater.key_schema
/**
* Fetch a doc's lines, version, ranges and recent ops from the
* document-updater.
*
* @param {string} projectId
* @param {string} docId
* @param {number} fromVersion - return ops from this version onwards (-1 for
* no ops)
*/
async function getDocument(projectId, docId, fromVersion) {
const timer = new metrics.Timer('get-document')
const url = `${settings.apis.documentupdater.url}/project/${projectId}/doc/${docId}?fromVersion=${fromVersion}&historyOTSupport=true`
@@ -67,11 +76,23 @@ async function getDocument(projectId, docId, fromVersion) {
}
}
/**
* Check that a doc is loadable from the document-updater.
*
* @param {string} projectId
* @param {string} docId
*/
async function checkDocument(projectId, docId) {
// in this call fromVersion = -1 means get document without docOps
return await getDocument(projectId, docId, -1)
}
/**
* Ask the document-updater to flush a project to mongo and remove it from
* redis.
*
* @param {string} projectId
*/
async function flushProjectToMongoAndDelete(projectId) {
// this method is called when the last connected user leaves the project
logger.debug({ projectId }, 'deleting project from document updater')
@@ -98,6 +119,11 @@ async function flushProjectToMongoAndDelete(projectId) {
}
}
/**
* Pick a random shard of the document-updater dispatch list.
*
* @return {string}
*/
function _getPendingUpdateListKey() {
const shard = _.random(0, settings.pendingUpdateListShardCount - 1)
if (shard === 0) {
@@ -107,7 +133,22 @@ function _getPendingUpdateListKey() {
}
}
async function queueChange(projectId, docId, change) {
/**
* Queue an update for processing by the document-updater.
*
* @param {string} projectId
* @param {string} docId
* @param {Object} change
* @param {boolean} usePerProjectPendingUpdates - feature flag assigned on
* joinProject; in migration phase 2 it selects the per-project queue
* over the legacy per-doc queue
*/
async function queueChange(
projectId,
docId,
change,
usePerProjectPendingUpdates
) {
const allowedKeys = ['doc', 'op', 'v', 'dupIfSource', 'meta', 'lastV', 'hash']
change = _.pick(change, allowedKeys)
const jsonChange = JSON.stringify(change)
@@ -121,8 +162,44 @@ async function queueChange(projectId, docId, change) {
throw new UpdateTooLargeError(updateSize)
}
switch (settings.pendingUpdatesMigrationPhase) {
case 1:
await _queueChangeOnDocQueue(projectId, docId, jsonChange)
break
case 2:
// the consumer drains both queues in phase 2, so the feature flag can
// safely pick the queue per client
if (usePerProjectPendingUpdates) {
await _queueChangeOnProjectQueue(projectId, jsonChange)
} else {
await _queueChangeOnDocQueue(projectId, docId, jsonChange)
}
break
case 3:
await _queueChangeOnProjectQueue(projectId, jsonChange)
break
default:
throw new OError('invalid pendingUpdatesMigrationPhase', {
pendingUpdatesMigrationPhase: settings.pendingUpdatesMigrationPhase,
})
}
}
/**
* Queue an update on the legacy per-doc queue (phase 1): push the payload
* onto the doc queue, then put a `project_id:doc_id` marker onto
* pending-updates-list.
*
* @param {string} projectId
* @param {string} docId
* @param {string} jsonChange - the serialized change
*/
async function _queueChangeOnDocQueue(projectId, docId, jsonChange) {
// record metric for each update added to queue
metrics.summary('redis.pendingUpdates', updateSize, { status: 'push' })
metrics.summary('redis.pendingUpdates', jsonChange.length, {
status: 'push',
path: 'doc',
})
const docKey = `${projectId}:${docId}`
// Push onto pendingUpdates for doc_id first, because once the doc updater
@@ -143,6 +220,40 @@ async function queueChange(projectId, docId, change) {
}
}
/**
* Queue an update on the per-project queue (phases 2 & 3): push the payload
* (tagged with its doc id) onto the project queue, then put a bare
* `project_id` marker onto pending-updates-list.
*
* @param {string} projectId
* @param {string} jsonChange
*/
async function _queueChangeOnProjectQueue(projectId, jsonChange) {
// record metric for each update added to queue
metrics.summary('redis.pendingUpdates', jsonChange.length, {
status: 'push',
path: 'project',
})
try {
await rclient.rpush(
Keys.pendingProjectUpdates({ project_id: projectId }),
jsonChange
)
} catch (error) {
throw new OError('error pushing project update into redis').withCause(error)
}
const queueKey = _getPendingUpdateListKey()
try {
await rclient.rpush(queueKey, projectId)
} catch (error) {
throw new OError('error pushing project_id into redis')
.withInfo({ queueKey })
.withCause(error)
}
}
export default {
getDocument: callbackify(getDocument),
checkDocument: callbackify(checkDocument),
services/real-time/app/js/WebApiManager.js
+1
View File
@@ -54,6 +54,7 @@ async function joinProject(projectId, user) {
isRestrictedUser: data.isRestrictedUser,
isTokenMember: data.isTokenMember,
isInvitedMember: data.isInvitedMember,
usePerProjectPendingUpdates: data.usePerProjectPendingUpdates,
}
return {
project: data.project,
services/real-time/app/js/WebsocketController.js
+8 -1
View File
@@ -100,6 +100,8 @@ export default WebsocketController = {
client.ol_context.is_restricted_user = !!userMetadata.isRestrictedUser
client.ol_context.is_token_member = !!userMetadata.isTokenMember
client.ol_context.is_invited_member = !!userMetadata.isInvitedMember
client.ol_context.use_per_project_pending_updates =
!!userMetadata.usePerProjectPendingUpdates
RoomManager.joinProject(client, projectId, function (err) {
if (err) {
@@ -557,7 +559,11 @@ export default WebsocketController = {
applyOtUpdate(client, docId, update, callback) {
// client may have disconnected, but we can submit their update to doc-updater anyways.
const { user_id: userId, project_id: projectId } = client.ol_context
const {
user_id: userId,
project_id: projectId,
use_per_project_pending_updates: usePerProjectPendingUpdates,
} = client.ol_context
if (!projectId) {
return callback(new NotJoinedError())
}
@@ -607,6 +613,7 @@ export default WebsocketController = {
projectId,
docId,
update,
usePerProjectPendingUpdates,
function (error) {
if ((error && error.message) === 'update is too large') {
metrics.inc('update_too_large')
services/real-time/config/settings.defaults.cjs
+17
View File
@@ -64,6 +64,9 @@ const settings = {
pendingUpdates({ doc_id }) {
return `PendingUpdates:{${doc_id}}`
},
pendingProjectUpdates({ project_id }) {
return `PendingProjectUpdates:{${project_id}}`
},
},
maxRetriesPerRequest: parseInt(
process.env.DOC_UPDATER_REDIS_MAX_RETRIES_PER_REQUEST ||
@@ -131,6 +134,20 @@ const settings = {
10
),
// Migration of the document-updater update queue from a per-doc queue to a
// per-project queue. Must be kept in step with document-updater.
// phase 1: write the legacy per-doc queue (`PendingUpdates:{doc_id}` +
// `<project_id>:<doc_id>` marker)
// phase 2: write the per-project queue (`PendingProjectUpdates:{project_id}`
// + bare `<project_id>` marker) for clients with the
// per-project-pending-updates feature flag (assigned by web on
// joinProject), the legacy per-doc queue otherwise
// phase 3: write the per-project queue for all clients
pendingUpdatesMigrationPhase: parseInt(
process.env.PENDING_UPDATES_MIGRATION_PHASE || '1',
10
),
// combine
// max_doc_length (2mb see above) * 2 (delete + insert)
// max_ranges_size (3mb see MAX_RANGES_SIZE in document-updater)
services/real-time/test/acceptance/js/helpers/MockWebServer.js
+2
View File
@@ -74,6 +74,8 @@ export default MockWebServer = {
isRestrictedUser: !!userMetadata?.isRestrictedUser,
isTokenMember: !!userMetadata?.isTokenMember,
isInvitedMember: !!userMetadata?.isInvitedMember,
usePerProjectPendingUpdates:
!!userMetadata?.usePerProjectPendingUpdates,
})
}
)
services/real-time/test/unit/js/DocumentUpdaterManager.test.js
+102 -1
View File
@@ -24,11 +24,15 @@ describe('DocumentUpdaterManager', function () {
pendingUpdates({ doc_id: docId }) {
return `PendingUpdates:${docId}`
},
pendingProjectUpdates({ project_id: projectId }) {
return `PendingProjectUpdates:${projectId}`
},
},
},
},
maxUpdateSize: 7 * 1024 * 1024,
pendingUpdateListShardCount: 10,
pendingUpdatesMigrationPhase: 1,
}
ctx.rclient = { auth() {} }
ctx.fetchJson = sinon.stub()
@@ -239,7 +243,7 @@ describe('DocumentUpdaterManager', function () {
describe('queueChange', function () {
beforeEach(function (ctx) {
ctx.change = {
doc: '1234567890',
doc: ctx.doc_id,
op: [{ d: 'test', p: 345 }],
v: 789,
}
@@ -274,6 +278,103 @@ describe('DocumentUpdaterManager', function () {
})
})
describe('in migration phase 2 with the per-project queue flag', function () {
beforeEach(async function (ctx) {
ctx.settings.pendingUpdatesMigrationPhase = 2
await ctx.DocumentUpdaterManager.promises.queueChange(
ctx.project_id,
ctx.doc_id,
ctx.change,
true
)
})
it('should push the change onto the per-project queue with its doc id', function (ctx) {
ctx.rclient.rpush
.calledWith(
`PendingProjectUpdates:${ctx.project_id}`,
JSON.stringify(ctx.change)
)
.should.equal(true)
})
it('should not push onto the legacy per-doc queue', function (ctx) {
ctx.rclient.rpush
.calledWith(`PendingUpdates:${ctx.doc_id}`)
.should.equal(false)
})
it('should notify the doc updater with a bare project id marker', function (ctx) {
const secondCall = ctx.rclient.rpush.secondCall
secondCall.should.exist
const queueKey = secondCall.args[0]
queueKey.should.match(/^pending-updates-list(-\d+)?$/)
secondCall.args[1].should.equal(ctx.project_id)
})
})
describe('in migration phase 2 without the per-project queue flag', function () {
beforeEach(async function (ctx) {
ctx.settings.pendingUpdatesMigrationPhase = 2
await ctx.DocumentUpdaterManager.promises.queueChange(
ctx.project_id,
ctx.doc_id,
ctx.change,
false
)
})
it('should push the change onto the legacy per-doc queue', function (ctx) {
ctx.rclient.rpush
.calledWith(
`PendingUpdates:${ctx.doc_id}`,
JSON.stringify(ctx.change)
)
.should.equal(true)
})
it('should not push onto the per-project queue', function (ctx) {
ctx.rclient.rpush
.calledWith(`PendingProjectUpdates:${ctx.project_id}`)
.should.equal(false)
})
it('should notify the doc updater with a project_id:doc_id marker', function (ctx) {
const secondCall = ctx.rclient.rpush.secondCall
secondCall.should.exist
const queueKey = secondCall.args[0]
queueKey.should.match(/^pending-updates-list(-\d+)?$/)
secondCall.args[1].should.equal(`${ctx.project_id}:${ctx.doc_id}`)
})
})
describe('in migration phase 3 (per-project queue only)', function () {
beforeEach(async function (ctx) {
ctx.settings.pendingUpdatesMigrationPhase = 3
await ctx.DocumentUpdaterManager.promises.queueChange(
ctx.project_id,
ctx.doc_id,
ctx.change,
false
)
})
it('should push the change onto the per-project queue even without the flag', function (ctx) {
ctx.rclient.rpush
.calledWith(
`PendingProjectUpdates:${ctx.project_id}`,
JSON.stringify(ctx.change)
)
.should.equal(true)
})
it('should not push onto the legacy per-doc queue', function (ctx) {
ctx.rclient.rpush
.calledWith(`PendingUpdates:${ctx.doc_id}`)
.should.equal(false)
})
})
describe('with error talking to redis during rpush', function () {
beforeEach(function (ctx) {
ctx.rclient.rpush = sinon
services/real-time/test/unit/js/WebApiManager.test.js
+6
View File
@@ -42,6 +42,7 @@ describe('WebApiManager', function () {
isRestrictedUser: true,
isTokenMember: true,
isInvitedMember: true,
usePerProjectPendingUpdates: true,
}
ctx.fetchUtils.fetchJson.resolves(ctx.response)
ctx.WebApiManager.joinProject(ctx.project_id, ctx.user, ctx.callback)
@@ -70,6 +71,8 @@ describe('WebApiManager', function () {
isRestrictedUser: ctx.response.isRestrictedUser,
isTokenMember: ctx.response.isTokenMember,
isInvitedMember: ctx.response.isInvitedMember,
usePerProjectPendingUpdates:
ctx.response.usePerProjectPendingUpdates,
})
.should.equal(true)
})
@@ -89,6 +92,7 @@ describe('WebApiManager', function () {
isRestrictedUser: true,
isTokenMember: false,
isInvitedMember: false,
usePerProjectPendingUpdates: false,
}
ctx.fetchUtils.fetchJson.resolves(ctx.response)
ctx.WebApiManager.joinProject(ctx.project_id, ctx.user, ctx.callback)
@@ -120,6 +124,8 @@ describe('WebApiManager', function () {
isRestrictedUser: ctx.response.isRestrictedUser,
isTokenMember: ctx.response.isTokenMember,
isInvitedMember: ctx.response.isInvitedMember,
usePerProjectPendingUpdates:
ctx.response.usePerProjectPendingUpdates,
}
)
})
services/real-time/test/unit/js/WebsocketController.test.js
+15 -5
View File
@@ -97,12 +97,14 @@ describe('WebsocketController', function () {
ctx.isRestrictedUser = true
ctx.isTokenMember = true
ctx.isInvitedMember = true
ctx.usePerProjectPendingUpdates = true
ctx.WebApiManager.joinProject = sinon
.stub()
.callsArgWith(2, null, ctx.project, ctx.privilegeLevel, {
isRestrictedUser: ctx.isRestrictedUser,
isTokenMember: ctx.isTokenMember,
isInvitedMember: ctx.isInvitedMember,
usePerProjectPendingUpdates: ctx.usePerProjectPendingUpdates,
})
ctx.RoomManager.joinProject = sinon.stub().callsArg(2)
return ctx.WebsocketController.joinProject(
@@ -176,6 +178,11 @@ describe('WebsocketController', function () {
ctx.isInvitedMember
)
})
it('should set the use_per_project_pending_updates flag on the client', function (ctx) {
ctx.client.ol_context.use_per_project_pending_updates.should.equal(
ctx.usePerProjectPendingUpdates
)
})
it('should call the callback with the project, privilegeLevel and protocolVersion', function (ctx) {
return ctx.callback
.calledWith(
@@ -240,12 +247,14 @@ describe('WebsocketController', function () {
ctx.isRestrictedUser = true
ctx.isTokenMember = true
ctx.isInvitedMember = true
ctx.usePerProjectPendingUpdates = true
ctx.WebApiManager.joinProject = sinon
.stub()
.callsArgWith(2, null, ctx.project, ctx.privilegeLevel, {
isRestrictedUser: ctx.isRestrictedUser,
isTokenMember: ctx.isTokenMember,
isInvitedMember: ctx.isInvitedMember,
usePerProjectPendingUpdates: ctx.usePerProjectPendingUpdates,
})
ctx.RoomManager.joinProject = sinon
.stub()
@@ -1466,10 +1475,11 @@ describe('WebsocketController', function () {
ctx.update = { op: { p: 12, t: 'foo' } }
ctx.client.ol_context.user_id = ctx.user_id
ctx.client.ol_context.project_id = ctx.project_id
ctx.client.ol_context.use_per_project_pending_updates = true
ctx.WebsocketController._assertClientCanApplyUpdate = sinon
.stub()
.yields()
ctx.DocumentUpdaterManager.queueChange = sinon.stub().callsArg(3)
ctx.DocumentUpdaterManager.queueChange = sinon.stub().callsArg(4)
})
describe('succesfully', function () {
@@ -1490,9 +1500,9 @@ describe('WebsocketController', function () {
ctx.update.meta.user_id.should.equal(ctx.user_id)
})
it('should queue the update', function (ctx) {
it('should queue the update with the per-project queue flag', function (ctx) {
ctx.DocumentUpdaterManager.queueChange
.calledWith(ctx.project_id, ctx.doc_id, ctx.update)
.calledWith(ctx.project_id, ctx.doc_id, ctx.update, true)
.should.equal(true)
})
@@ -1510,7 +1520,7 @@ describe('WebsocketController', function () {
ctx.client.disconnect = sinon.stub()
ctx.DocumentUpdaterManager.queueChange = sinon
.stub()
.callsArgWith(3, (ctx.error = new Error('Something went wrong')))
.callsArgWith(4, (ctx.error = new Error('Something went wrong')))
ctx.WebsocketController.applyOtUpdate(
ctx.client,
ctx.doc_id,
@@ -1570,7 +1580,7 @@ describe('WebsocketController', function () {
const error = new UpdateTooLargeError(7372835)
ctx.DocumentUpdaterManager.queueChange = sinon
.stub()
.callsArgWith(3, error)
.callsArgWith(4, error)
ctx.WebsocketController.applyOtUpdate(
ctx.client,
ctx.doc_id,
services/web/app/src/Features/Editor/EditorHttpController.mjs
+11
View File
@@ -7,6 +7,7 @@ import Metrics from '@overleaf/metrics'
import CollaboratorsInviteGetter from '../Collaborators/CollaboratorsInviteGetter.mjs'
import PrivilegeLevels from '../Authorization/PrivilegeLevels.mjs'
import SessionManager from '../Authentication/SessionManager.mjs'
import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs'
import Errors from '../Errors/Errors.js'
import { expressify } from '@overleaf/promise-utils'
import Settings from '@overleaf/settings'
@@ -67,12 +68,22 @@ async function joinProject(req, res, next) {
)
}
let usePerProjectPendingUpdates = false
if (userId) {
usePerProjectPendingUpdates =
await SplitTestHandler.promises.featureFlagEnabledForUser(
userId,
'per-project-pending-updates'
)
}
res.json({
project,
privilegeLevel,
isRestrictedUser,
isTokenMember,
isInvitedMember,
usePerProjectPendingUpdates,
})
}
services/web/scripts/fix_oversized_docs.mjs
+6 -1
View File
@@ -152,7 +152,12 @@ async function deleteDocFromRedis(projectId, docId) {
`lastUpdatedAt:{${docId}}`,
`lastUpdatedBy:{${docId}}`
)
await redis.srem(`DocsIn:{${projectId}}`, projectId)
// NOTE: once the update-queue migration reaches phase >= 2, a doc's pending
// updates live in the shared `PendingProjectUpdates:{projectId}` list (on the
// document-updater redis) rather than `PendingUpdates:{docId}`. That list is
// per-project, so it cannot be cleared here without dropping other docs'
// updates - flush/drain the project before running this script.
await redis.srem(`DocsIn:{${projectId}}`, docId)
}
try {
services/web/scripts/recover_docs_from_redis.mjs
+6 -1
View File
@@ -170,7 +170,12 @@ async function deleteDocFromRedis(projectId, docId) {
`lastUpdatedAt:{${docId}}`,
`lastUpdatedBy:{${docId}}`
)
await redis.srem(`DocsIn:{${projectId}}`, projectId)
// NOTE: once the update-queue migration reaches phase >= 2, a doc's pending
// updates live in the shared `PendingProjectUpdates:{projectId}` list (on the
// document-updater redis) rather than `PendingUpdates:{docId}`. That list is
// per-project, so it cannot be cleared here without dropping other docs'
// updates - flush/drain the project before running this script.
await redis.srem(`DocsIn:{${projectId}}`, docId)
}
try {
services/web/test/unit/src/Editor/EditorHttpController.test.mjs
+35
View File
@@ -160,6 +160,7 @@ describe('EditorHttpController', function () {
ctx.SplitTestHandler = {
promises: {
getAssignmentForUser: sinon.stub().resolves({ variant: 'default' }),
featureFlagEnabledForUser: sinon.stub().resolves(false),
},
}
ctx.UserGetter = { promises: { getUser: sinon.stub().resolves(null, {}) } }
@@ -304,6 +305,7 @@ describe('EditorHttpController', function () {
isRestrictedUser: false,
isTokenMember: false,
isInvitedMember: true,
usePerProjectPendingUpdates: false,
})
})
@@ -361,6 +363,7 @@ describe('EditorHttpController', function () {
isRestrictedUser: true,
isTokenMember: false,
isInvitedMember: false,
usePerProjectPendingUpdates: false,
})
})
})
@@ -417,8 +420,14 @@ describe('EditorHttpController', function () {
isRestrictedUser: true,
isTokenMember: false,
isInvitedMember: false,
usePerProjectPendingUpdates: false,
})
})
it('should not fetch a split test assignment', function (ctx) {
expect(ctx.SplitTestHandler.promises.featureFlagEnabledForUser).not.to
.have.been.called
})
})
describe('with a token access user', function () {
@@ -451,10 +460,36 @@ describe('EditorHttpController', function () {
isRestrictedUser: false,
isTokenMember: true,
isInvitedMember: false,
usePerProjectPendingUpdates: false,
})
})
})
describe('with the per-project-pending-updates flag enabled', function () {
beforeEach(async function (ctx) {
ctx.SplitTestHandler.promises.featureFlagEnabledForUser.resolves(true)
await new Promise(resolve => {
ctx.res.callback = resolve
ctx.EditorHttpController.joinProject(ctx.req, ctx.res)
})
})
it('should fetch the assignment for the joining user', function (ctx) {
expect(
ctx.SplitTestHandler.promises.featureFlagEnabledForUser
).to.have.been.calledWith(
ctx.user._id.toString(),
'per-project-pending-updates'
)
})
it('should return usePerProjectPendingUpdates=true', function (ctx) {
expect(ctx.res.json).toHaveBeenCalledWith(
expect.objectContaining({ usePerProjectPendingUpdates: true })
)
})
})
describe('when project is not found', function () {
beforeEach(async function (ctx) {
ctx.ProjectGetter.promises.getProject.resolves(null)