agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-14 14:13:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

[monorepo] clean up per-project queue migration (#35573)

Browse Source 
GitOrigin-RevId: 988df4cdc7d929efc921e7e99d803c6ce6e52c57
This commit is contained in:
Jakob Ackermann
2026-09-17 11:31:52 +00:00
committed by Copybot
parent d8f167dec2
commit 68352783b0
27 changed files with 195 additions and 1528 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/web/scripts/fix_oversized_docs.mjs
+34 -11
View File
@@ -15,7 +15,7 @@ const redis = RedisWrapper.createClient(Settings.redis.web)
function parseArgs() {
const args = minimist(process.argv.slice(2), {
boolean: ['commit', 'ignore-ranges'],
boolean: ['commit', 'ignore-ranges', 'ignore-pending-updates'],
})
const projectIds = args._
@@ -23,20 +23,22 @@ function parseArgs() {
console.log(`Usage: ${process.argv[1]} [OPTS] PROJECT_ID
Options:
--commit Actually convert oversized docs to binary files
--max-doc-size Size over which docs are converted to binary files
--ignore-ranges Convert docs even if they contain ranges
--commit Actually convert oversized docs to binary files
--max-doc-size Size over which docs are converted to binary files
--ignore-ranges Convert docs even if they contain ranges
--ignore-pending-updates Process projects even if they have pending updates
`)
process.exit(0)
}
const commit = args.commit
const ignoreRanges = args['ignore-ranges']
const ignorePendingUpdates = args['ignore-pending-updates']
const maxDocSize = args['max-doc-size']
? parseInt(args['max-doc-size'], 10)
: 2 * 1024 * 1024
return { projectIds, commit, ignoreRanges, maxDocSize }
return { projectIds, commit, ignoreRanges, ignorePendingUpdates, maxDocSize }
}
async function main() {
@@ -49,12 +51,36 @@ async function main() {
}
async function processProject(projectId) {
await checkPendingUpdates(projectId)
const docIds = await getDocIds(projectId)
for (const docId of docIds) {
await processDoc(projectId, docId)
}
}
// A doc's pending updates live in the shared
// `PendingProjectUpdates:{projectId}` list (on the document-updater redis).
// They would be lost when converting the doc to a binary file, so refuse to
// process a project that has pending updates - flush/drain the project first,
// or override with --ignore-pending-updates.
async function checkPendingUpdates(projectId) {
const pendingUpdates = await redis.llen(
`PendingProjectUpdates:{${projectId}}`
)
if (pendingUpdates === 0) {
return
}
if (opts.ignorePendingUpdates) {
console.log(
`Ignoring ${pendingUpdates} pending update(s) in project ${projectId}`
)
} else {
throw new Error(
`project ${projectId} has ${pendingUpdates} pending update(s) in redis - flush/drain the project first, or re-run with --ignore-pending-updates`
)
}
}
async function processDoc(projectId, docId) {
const doc = await getDoc(projectId, docId)
const size = doc.lines.reduce((sum, line) => sum + line.length + 1, 0)
@@ -148,15 +174,12 @@ async function deleteDocFromRedis(projectId, docId) {
`UnflushedTime:{${docId}}`,
`Pathname:{${docId}}`,
`ProjectHistoryId:{${docId}}`,
`PendingUpdates:{${docId}}`,
`lastUpdatedAt:{${docId}}`,
`lastUpdatedBy:{${docId}}`
)
// NOTE: once the update-queue migration reaches phase >= 2, a doc's pending
// updates live in the shared `PendingProjectUpdates:{projectId}` list (on the
// document-updater redis) rather than `PendingUpdates:{docId}`. That list is
// per-project, so it cannot be cleared here without dropping other docs'
// updates - flush/drain the project before running this script.
// NOTE: the shared `PendingProjectUpdates:{projectId}` list is per-project
// and cannot be cleared here without dropping other docs' updates - see
// checkPendingUpdates above.
await redis.srem(`DocsIn:{${projectId}}`, docId)
}
services/web/scripts/recover_docs_from_redis.mjs
+28 -7
View File
@@ -14,13 +14,15 @@ const redis = RedisWrapper.createClient(Settings.redis.web)
function parseArgs() {
const args = minimist(process.argv.slice(2), {
boolean: ['commit'],
boolean: ['commit', 'ignore-pending-updates'],
})
const commit = args.commit
const deleteOversized = args['delete-oversized']
const ignorePendingUpdates = args['ignore-pending-updates']
return {
commit,
deleteOversized,
ignorePendingUpdates,
maxDocSize: 2 * 1024 * 1024,
}
}
@@ -79,6 +81,28 @@ async function processDoc(docId) {
)
return
}
// A doc's pending updates live in the shared
// `PendingProjectUpdates:{projectId}` list (on the document-updater redis).
// Restoring the doc - or deleting it when oversized - would drop them, so
// skip the doc while the project has pending updates: flush/drain the
// project first, or override with --ignore-pending-updates.
const pendingUpdates = await redis.llen(
`PendingProjectUpdates:{${projectId}}`
)
if (pendingUpdates > 0) {
if (opts.ignorePendingUpdates) {
logger.warn(
{ docId, projectId, pendingUpdates },
'ignoring pending updates for project'
)
} else {
logger.warn(
{ docId, projectId, pendingUpdates },
'project has pending updates - skipping doc (flush/drain the project or use --ignore-pending-updates)'
)
return
}
}
// if the doc is too big we will need to convert it to a file, skip it for now
// or delete the doc if the --delete-oversized option is used
const size = doc.lines.reduce((sum, line) => sum + line.length + 1, 0)
@@ -166,15 +190,12 @@ async function deleteDocFromRedis(projectId, docId) {
`UnflushedTime:{${docId}}`,
`Pathname:{${docId}}`,
`ProjectHistoryId:{${docId}}`,
`PendingUpdates:{${docId}}`,
`lastUpdatedAt:{${docId}}`,
`lastUpdatedBy:{${docId}}`
)
// NOTE: once the update-queue migration reaches phase >= 2, a doc's pending
// updates live in the shared `PendingProjectUpdates:{projectId}` list (on the
// document-updater redis) rather than `PendingUpdates:{docId}`. That list is
// per-project, so it cannot be cleared here without dropping other docs'
// updates - flush/drain the project before running this script.
// NOTE: the shared `PendingProjectUpdates:{projectId}` list is per-project
// and cannot be cleared here without dropping other docs' updates - the
// pending-updates check in processDoc guards against this.
await redis.srem(`DocsIn:{${projectId}}`, docId)
}