From 2797c6e16a1201a88bac13a4751dbe5d4b347153 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 21 Jan 2025 16:41:37 +0000 Subject: [PATCH] [web] extend scripts for finding and fixing broken file-trees (#22984) - Validate _id fields to be ObjectIds - Include more debug output, notably include the folder/doc/file id - Emit and consume JSON output - Read broken file-tree details from (ad-hoc) file - Use generator instead of accumulating all results per project first - Use batchedUpdate to be able to pause and resume processing GitOrigin-RevId: 606ac431fff65891e09479c0ba9bcb7e7dc5cbe4 --- .../web/scripts/find_malformed_filetrees.mjs | 156 ++++++++++++------ .../web/scripts/fix_malformed_filetree.mjs | 102 +++++++++--- 2 files changed, 188 insertions(+), 70 deletions(-) diff --git a/services/web/scripts/find_malformed_filetrees.mjs b/services/web/scripts/find_malformed_filetrees.mjs index a5cae0bda4..25b4a77a59 100644 --- a/services/web/scripts/find_malformed_filetrees.mjs +++ b/services/web/scripts/find_malformed_filetrees.mjs @@ -1,107 +1,163 @@ -import { - db, - READ_PREFERENCE_SECONDARY, -} from '../app/src/infrastructure/mongodb.js' +// @ts-check +import { db, ObjectId } from '../app/src/infrastructure/mongodb.js' +import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' + +/** + * @typedef {Object} Doc + * @property {ObjectId} _id + * @property {string} name + */ + +/** + * @typedef {Object} FileRef + * @property {ObjectId} _id + * @property {string} name + * @property {string} hash + */ + +/** + * @typedef {Object} Folder + * @property {ObjectId} _id + * @property {string} name + * @property {Array} docs + * @property {Array} folders + * @property {Array} fileRefs + */ + +/** + * @typedef {Object} Project + * @property {ObjectId} _id + * @property {Array} rootFolder + */ async function main() { - const projects = db.projects.find( - {}, - { - projection: { rootFolder: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) let projectsProcessed = 0 - for await (const project of projects) { - projectsProcessed += 1 - if (projectsProcessed % 100000 === 0) { - console.log(projectsProcessed, 'projects processed') - } - processProject(project) - } + await batchedUpdate( + db.projects, + {}, + /** + * @param {Array} projects + * @return {Promise} + */ + async function projects(projects) { + for (const project of projects) { + projectsProcessed += 1 + if (projectsProcessed % 100000 === 0) { + console.log(projectsProcessed, 'projects processed') + } + const projectId = project._id.toString() + for (const { reason, path, _id } of processProject(project)) { + console.log( + JSON.stringify({ + msg: 'bad file-tree path', + projectId, + reason, + path, + _id, + }) + ) + } + } + }, + { _id: 1, rootFolder: 1 } + ) } -function processProject(project) { +/** + * @param {Project} project + * @return {Generator<{path: string, reason: string, _id: any}, void, *>} + */ +function* processProject(project) { if (!project.rootFolder || !Array.isArray(project.rootFolder)) { - console.log('BAD PATH:', project._id, 'rootFolder') - return - } - if (!project.rootFolder[0]) { - console.log('BAD PATH:', project._id, 'rootFolder.0') - return - } - const badPaths = findBadPaths(project.rootFolder[0]) - for (const path of badPaths) { - console.log('BAD PATH:', project._id, `rootFolder.0.${path}`) + yield { reason: 'bad rootFolder', path: 'rootFolder', _id: null } + } else if (!project.rootFolder[0]) { + yield { reason: 'missing rootFolder', path: 'rootFolder.0', _id: null } + } else { + for (const { path, reason, _id } of findBadPaths(project.rootFolder[0])) { + yield { reason, path: `rootFolder.0${path}`, _id } + } } } -function findBadPaths(folder) { - const result = [] +/** + * @param {Folder} folder + * @return {Generator<{path: string, reason: string, _id: any}, void, *>} + */ +function* findBadPaths(folder) { + const folderId = folder._id - if (!folder._id) { - result.push('_id') + if (!(folderId instanceof ObjectId)) { + yield { path: '._id', reason: 'bad folder id', _id: folderId } } if (typeof folder.name !== 'string' || !folder.name) { - result.push('name') + yield { path: '.name', reason: 'bad folder name', _id: folderId } } if (folder.folders && Array.isArray(folder.folders)) { for (const [i, subfolder] of folder.folders.entries()) { if (!subfolder || typeof subfolder !== 'object') { - result.push(`folders.${i}`) + yield { path: `.folders.${i}`, reason: 'bad folder', _id: folderId } continue } - for (const badPath of findBadPaths(subfolder)) { - result.push(`folders.${i}.${badPath}`) + for (const { path, reason, _id } of findBadPaths(subfolder)) { + yield { path: `.folders.${i}${path}`, reason, _id } } } } else { - result.push('folders') + yield { path: '.folders', reason: 'missing .folders', _id: folderId } } if (folder.docs && Array.isArray(folder.docs)) { for (const [i, doc] of folder.docs.entries()) { if (!doc || typeof doc !== 'object') { - result.push(`docs.${i}`) + yield { path: `.docs.${i}`, reason: 'bad doc', _id: folderId } continue } - if (!doc._id) { - result.push(`docs.${i}._id`) + const docId = doc._id + if (!(docId instanceof ObjectId)) { + yield { path: `.docs.${i}._id`, reason: 'bad doc id', _id: docId } // no need to check further: this doc can be deleted continue } if (typeof doc.name !== 'string' || !doc.name) { - result.push(`docs.${i}.name`) + yield { path: `.docs.${i}.name`, reason: 'bad doc name', _id: docId } } } } else { - result.push('docs') + yield { path: '.docs', reason: 'missing .docs', _id: folderId } } if (folder.fileRefs && Array.isArray(folder.fileRefs)) { for (const [i, file] of folder.fileRefs.entries()) { if (!file || typeof file !== 'object') { - result.push(`fileRefs.${i}`) + yield { path: `.fileRefs.${i}`, reason: 'bad file', _id: folderId } continue } - if (!file._id) { - result.push(`fileRefs.${i}._id`) + const fileId = file._id + if (!(fileId instanceof ObjectId)) { + yield { path: `.fileRefs.${i}._id`, reason: 'bad file id', _id: fileId } // no need to check further: this file can be deleted continue } if (typeof file.name !== 'string' || !file.name) { - result.push(`fileRefs.${i}.name`) + yield { + path: `.fileRefs.${i}.name`, + reason: 'bad file name', + _id: fileId, + } } if (typeof file.hash !== 'string' || !file.hash) { - result.push(`fileRefs.${i}.hash`) + yield { + path: `.fileRefs.${i}.hash`, + reason: 'bad file hash', + _id: fileId, + } } } } else { - result.push('fileRefs') + yield { path: '.fileRefs', reason: 'missing .fileRefs', _id: folderId } } - return result } try { diff --git a/services/web/scripts/fix_malformed_filetree.mjs b/services/web/scripts/fix_malformed_filetree.mjs index 2caf26aac7..5e12c3cb77 100644 --- a/services/web/scripts/fix_malformed_filetree.mjs +++ b/services/web/scripts/fix_malformed_filetree.mjs @@ -1,18 +1,73 @@ /** * This script fixes problems found by the find_malformed_filetrees.js script. * - * The script takes two arguments: the project id and the problemtatic path. - * This is the output format of each line in the find_malformed_filetrees.js - * script. + * The script takes a single argument --logs pointing at the output of a + * previous run of the find_malformed_filetrees.js script. + * + * Alternatively, use an adhoc file: --logs=<(echo '{"projectId":"...","path":"..."}') */ import mongodb from 'mongodb-legacy' import { db } from '../app/src/infrastructure/mongodb.js' import ProjectLocator from '../app/src/Features/Project/ProjectLocator.js' +import minimist from 'minimist' +import readline from 'node:readline' +import fs from 'node:fs' +import logger from '@overleaf/logger' const { ObjectId } = mongodb +const argv = minimist(process.argv.slice(2), { + string: ['logs'], +}) + +let gracefulShutdownInitiated = false + +process.on('SIGINT', handleSignal) +process.on('SIGTERM', handleSignal) + +function handleSignal() { + gracefulShutdownInitiated = true + console.warn('graceful shutdown initiated, draining queue') +} + +const STATS = { + processedLines: 0, + success: 0, + alreadyProcessed: 0, + hash: 0, + failed: 0, + unmatched: 0, +} +function logStats() { + console.log( + JSON.stringify({ + time: new Date(), + gracefulShutdownInitiated, + ...STATS, + }) + ) +} +setInterval(logStats, 10_000) + async function main() { - const { projectId, mongoPath } = parseArgs() + const rl = readline.createInterface({ + input: fs.createReadStream(argv.logs), + }) + for await (const line of rl) { + if (gracefulShutdownInitiated) break + STATS.processedLines++ + if (!line.startsWith('{')) continue + try { + const { projectId, path, _id } = JSON.parse(line) + await processBadPath(projectId, path, _id) + } catch (err) { + STATS.failed++ + logger.err({ line, err }, 'failed to fix tree') + } + } +} + +async function processBadPath(projectId, mongoPath, _id) { let modifiedCount if (isRootFolder(mongoPath)) { modifiedCount = await fixRootFolder(projectId) @@ -30,27 +85,22 @@ async function main() { } else if (isName(mongoPath)) { modifiedCount = await fixName(projectId, mongoPath) } else if (isHash(mongoPath)) { - console.error(`Missing file hash: ${mongoPath}`) + console.error(`Missing file hash: ${projectId}/${_id} (${mongoPath})`) console.error('SaaS: likely needs filestore restore') console.error('Server Pro: please reach out to support') - process.exit(1) + STATS.hash++ + return } else { console.error(`Unexpected mongo path: ${mongoPath}`) - process.exit(1) + STATS.unmatched++ + return } - console.log(`${modifiedCount} project(s) modified`) - process.exit(0) -} - -function parseArgs() { - const args = process.argv.slice(2) - if (args.length !== 2) { - console.error('Usage: fix_malformed_filetree.js PROJECT_ID MONGO_PATH') - process.exit(1) + if (modifiedCount === 0) { + STATS.alreadyProcessed++ + } else { + STATS.success++ } - const [projectId, mongoPath] = args - return { projectId: new ObjectId(projectId), mongoPath } } function isRootFolder(path) { @@ -182,8 +232,20 @@ function findUniqueName(existingFilenames) { } try { - await main() - process.exit(0) + try { + await main() + } finally { + logStats() + } + if (STATS.failed > 0) { + process.exit(Math.min(STATS.failed, 99)) + } else if (STATS.hash > 0) { + process.exit(100) + } else if (STATS.unmatched > 0) { + process.exit(101) + } else { + process.exit(0) + } } catch (error) { console.error(error) process.exit(1)