diff --git a/libraries/eslint-plugin/index.js b/libraries/eslint-plugin/index.js index 6119735c05..8af8043233 100644 --- a/libraries/eslint-plugin/index.js +++ b/libraries/eslint-plugin/index.js @@ -22,5 +22,6 @@ module.exports = { 'sorted-keys-in-locales': require('./sorted-keys-in-locales'), 'locale-variables-match-en': require('./locale-variables-match-en'), 'no-orphan-locale-keys': require('./no-orphan-locale-keys'), + 'no-mocha-before': require('./no-mocha-before'), }, } diff --git a/libraries/eslint-plugin/no-mocha-before.js b/libraries/eslint-plugin/no-mocha-before.js new file mode 100644 index 0000000000..6166a968f9 --- /dev/null +++ b/libraries/eslint-plugin/no-mocha-before.js @@ -0,0 +1,112 @@ +const Path = require('node:path') + +/** + * Block bare mocha `before()` hooks in Cypress e2e suites. + * + * `before()` hooks run once per suite and do NOT re-run when a test retries, + * while `beforeEach`-based hooks (like `beforeWithReRunOnTestRetry`) do. + * Mixing the two also changes ordering: all `before()` hooks run ahead of + * every `beforeEach`, not in source order. This matters in the Server Pro + * e2e suite, where suites change the server config via `startWith()` while + * test setup runs in `beforeWithReRunOnTestRetry()`: when a later suite + * changes the config, both must re-run on retry and in registration order, + * otherwise a retried test runs against setup or config belonging to a + * different suite. + * + * The autofix rewrites `before(fn)` to `beforeWithReRunOnTestRetry(fn)` and + * adds the helper import when missing. Configure the helper location (repo + * root relative, no extension) via the `helperPath` option. + */ +module.exports = { + meta: { + type: 'suggestion', + docs: { + description: + 'Disallow mocha before() hooks in favor of beforeWithReRunOnTestRetry()', + }, + fixable: 'code', + schema: [ + { + type: 'object', + properties: { + helperPath: { type: 'string' }, + }, + additionalProperties: false, + }, + ], + messages: { + noMochaBefore: + 'before() hooks do not re-run when a test retries, so retried tests run against stale setup. Use beforeWithReRunOnTestRetry() instead.', + }, + }, + create(context) { + const helperPath = context.options[0]?.helperPath + let lastImport = null + let hasHelperImport = false + + function helperImportSpecifier() { + const absolute = Path.resolve(context.cwd, helperPath) + let relative = Path.relative( + Path.dirname(Path.resolve(context.cwd, context.filename)), + absolute + ) + relative = relative.split(Path.sep).join('/') + if (!relative.startsWith('.')) { + relative = `./${relative}` + } + return relative + } + + return { + ImportDeclaration(node) { + lastImport = node + if ( + node.specifiers.some( + specifier => + specifier.type === 'ImportSpecifier' && + specifier.imported.name === 'beforeWithReRunOnTestRetry' + ) + ) { + hasHelperImport = true + } + }, + CallExpression(node) { + if ( + node.callee.type !== 'Identifier' || + node.callee.name !== 'before' + ) { + return + } + // Only the global mocha hook; skip locally defined/imported `before` + const variable = context.sourceCode + .getScope(node) + .references.find(ref => ref.identifier === node.callee)?.resolved + if (variable && variable.defs.length > 0) { + return + } + + const fixable = node.arguments.length === 1 && helperPath + context.report({ + node: node.callee, + messageId: 'noMochaBefore', + fix: fixable + ? fixer => { + const fixes = [ + fixer.replaceText(node.callee, 'beforeWithReRunOnTestRetry'), + ] + if (!hasHelperImport) { + const importText = `import { beforeWithReRunOnTestRetry } from '${helperImportSpecifier()}'` + fixes.push( + lastImport + ? fixer.insertTextAfter(lastImport, `\n${importText}`) + : fixer.insertTextBeforeRange([0, 0], `${importText}\n`) + ) + } + return fixes + } + : undefined, + }) + }, + } + }, +} diff --git a/libraries/eslint-plugin/test/unit/rules.test.js b/libraries/eslint-plugin/test/unit/rules.test.js index 0d1158ffa5..0dc82ced44 100644 --- a/libraries/eslint-plugin/test/unit/rules.test.js +++ b/libraries/eslint-plugin/test/unit/rules.test.js @@ -14,6 +14,7 @@ import noConsecutiveSpacesInLocales from '../../no-consecutive-spaces-in-locales import noStraightApostrophesInLocales from '../../no-straight-apostrophes-in-locales.js' import frenchTypographyInLocales from '../../french-typography-in-locales.js' import sortedKeysInLocales from '../../sorted-keys-in-locales.js' +import noMochaBefore from '../../no-mocha-before.js' const __filename = fileURLToPath(import.meta.url) @@ -546,3 +547,81 @@ jsonRuleTester.run('french-typography-in-locales', frenchTypographyInLocales, { }, ], }) + +const noMochaBeforeOptions = [ + { helperPath: 'server-ce/test/helpers/beforeWithReRunOnTestRetry' }, +] + +ruleTester.run('no-mocha-before', noMochaBefore, { + valid: [ + { + code: `beforeEach(function () { setup() })`, + options: noMochaBeforeOptions, + }, + { + code: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry' +beforeWithReRunOnTestRetry(function () { setup() })`, + options: noMochaBeforeOptions, + }, + { + // locally defined `before` is not the mocha hook + code: `function before(fn) { fn() } +before(function () { setup() })`, + options: noMochaBeforeOptions, + }, + ], + invalid: [ + { + // rename + import added after the last existing import + filename: 'server-ce/test/foo.spec.ts', + code: `import { login } from './helpers/login' +before(async function () { this.timeout(1000) })`, + options: noMochaBeforeOptions, + errors: 1, + output: `import { login } from './helpers/login' +import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry' +beforeWithReRunOnTestRetry(async function () { this.timeout(1000) })`, + }, + { + // no imports yet: import inserted at the top + filename: 'server-ce/test/foo.spec.ts', + code: `before(function () { setup() })`, + options: noMochaBeforeOptions, + errors: 1, + output: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry' +beforeWithReRunOnTestRetry(function () { setup() })`, + }, + { + // helper already imported: rename only + filename: 'server-ce/test/foo.spec.ts', + code: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry' +before(function () { setup() })`, + options: noMochaBeforeOptions, + errors: 1, + output: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry' +beforeWithReRunOnTestRetry(function () { setup() })`, + }, + { + // file inside helpers/: relative path has no directory prefix + filename: 'server-ce/test/helpers/config.ts', + code: `before(function () { setup() })`, + options: noMochaBeforeOptions, + errors: 1, + output: `import { beforeWithReRunOnTestRetry } from './beforeWithReRunOnTestRetry' +beforeWithReRunOnTestRetry(function () { setup() })`, + }, + { + // two-argument mocha form: report without autofix + filename: 'server-ce/test/foo.spec.ts', + code: `before('named hook', function () { setup() })`, + options: noMochaBeforeOptions, + errors: 1, + }, + { + // no helperPath configured: report without autofix + filename: 'server-ce/test/foo.spec.ts', + code: `before(function () { setup() })`, + errors: 1, + }, + ], +}) diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs deleted file mode 100644 index 7c2f031cfe..0000000000 --- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs +++ /dev/null @@ -1,1546 +0,0 @@ -// @ts-check -import Events from 'node:events' -import fs from 'node:fs' -import Path from 'node:path' -import { performance } from 'node:perf_hooks' -import Stream from 'node:stream' -import { setTimeout } from 'node:timers/promises' -import { ObjectId } from 'mongodb' -import pLimit from 'p-limit' -import logger from '@overleaf/logger' -import { - batchedUpdate, - objectIdFromInput, - renderObjectId, -} from '@overleaf/mongo-utils/batchedUpdate.js' -import OError from '@overleaf/o-error' -import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js' -import { - BlobStore, - GLOBAL_BLOBS, - loadGlobalBlobs, - getProjectBlobsBatch, - getStringLengthOfFile, - makeBlobForFile, -} from '../lib/blob_store/index.js' -import { db } from '../lib/mongodb.js' -import commandLineArgs from 'command-line-args' -import readline from 'node:readline' - -// Silence warning. -Events.setMaxListeners(20) - -// Enable caching for ObjectId.toString() -ObjectId.cacheHexString = true - -/** - * @typedef {import("overleaf-editor-core").Blob} Blob - * @typedef {import("perf_hooks").EventLoopUtilization} EventLoopUtilization - * @typedef {import("mongodb").Collection} Collection - * @typedef {import("mongodb").Collection} ProjectsCollection - * @typedef {import("mongodb").Collection<{project:Project}>} DeletedProjectsCollection - * @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor - */ - -/** - * @typedef {Object} FileRef - * @property {ObjectId} _id - * @property {string} hash - */ - -/** - * @typedef {Object} Folder - * @property {Array} folders - * @property {Array} fileRefs - */ - -/** - * @typedef {Object} DeletedFileRef - * @property {ObjectId} _id - * @property {ObjectId} projectId - * @property {string} hash - */ - -/** - * @typedef {Object} Project - * @property {ObjectId} _id - * @property {Array} rootFolder - * @property {{history: {id: (number|string)}}} overleaf - */ - -/** - * @typedef {Object} QueueEntry - * @property {ProjectContext} ctx - * @property {string} cacheKey - * @property {string} [fileId] - * @property {string} path - * @property {string} [hash] - * @property {Blob} [blob] - */ - -/** - * Start and end for range. - * @type {Date} - */ -const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z') -const DEFAULT_BATCH_RANGE_START_DATE = PUBLIC_LAUNCH_DATE -const DEFAULT_BATCH_RANGE_END_DATE = new Date() - -function usesDefaultBatchRange() { - return ( - BATCH_RANGE_START === - objectIdFromInput( - DEFAULT_BATCH_RANGE_START_DATE.toISOString() - ).toString() && - BATCH_RANGE_END === - objectIdFromInput(DEFAULT_BATCH_RANGE_END_DATE.toISOString()).toString() - ) -} - -/** - * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean, PROCESS_BLOBS: boolean, DRY_RUN: boolean, OUTPUT_FILE: string, DISPLAY_REPORT: boolean, CONCURRENCY: number, CONCURRENT_BATCHES: number, RETRIES: number, RETRY_DELAY_MS: number, RETRY_FILESTORE_404: boolean, BUFFER_DIR_PREFIX: string, STREAM_HIGH_WATER_MARK: number, LOGGING_INTERVAL: number, SLEEP_BEFORE_EXIT: number }} - */ -function parseArgs() { - const DEFAULT_OUTPUT_FILE = `/var/log/overleaf/file-migration-${new Date() - .toISOString() - .replace(/[:.]/g, '_')}.log` - - const args = commandLineArgs([ - { name: 'help', alias: 'h', type: Boolean }, - { name: 'all', alias: 'a', type: Boolean }, - { name: 'projects', type: Boolean }, - { name: 'deleted-projects', type: Boolean }, - { name: 'skip-hashed-files', type: Boolean }, - { name: 'skip-existing-blobs', type: Boolean }, - { name: 'from-file', type: String, defaultValue: '' }, - { name: 'concurrency', type: Number, defaultValue: 10 }, - { name: 'concurrent-batches', type: Number, defaultValue: 1 }, - { name: 'stream-high-water-mark', type: Number, defaultValue: 1024 * 1024 }, - { name: 'retries', type: Number, defaultValue: 10 }, - { name: 'retry-delay-ms', type: Number, defaultValue: 100 }, - { name: 'retry-filestore-404', type: Boolean }, - { name: 'dry-run', alias: 'n', type: Boolean }, - { - name: 'output', - alias: 'o', - type: String, - defaultValue: DEFAULT_OUTPUT_FILE, - }, - { name: 'report', type: Boolean }, - { - name: 'BATCH_RANGE_START', - type: String, - defaultValue: PUBLIC_LAUNCH_DATE.toISOString(), - }, - { - name: 'BATCH_RANGE_END', - type: String, - defaultValue: new Date().toISOString(), - }, - { name: 'logging-id', type: String, defaultValue: '' }, - { name: 'logging-interval-ms', type: Number, defaultValue: 60_000 }, - { - name: 'buffer-dir-prefix', - type: String, - defaultValue: '/tmp/back_fill_file_hash-', - }, - { name: 'sleep-before-exit-ms', type: Number, defaultValue: 1_000 }, - ]) - - // If no arguments are provided, display a usage message - if (process.argv.length <= 2) { - console.error( - 'Usage: node back_fill_file_hash.mjs --all | --projects | --deleted-projects' - ) - process.exit(1) - } - - // If --help is provided, display the help message - if (args.help) { - console.log(`Usage: node back_fill_file_hash.mjs [options] - -Project selection options: - --all, -a Process all projects, including deleted ones - --projects Process projects (excluding deleted ones) - --deleted-projects Process deleted projects - --from-file Process selected projects ids from file - -File selection options: - --skip-hashed-files Skip processing files that already have a hash - --skip-existing-blobs Skip processing files already in the blob store - -Logging options: - --output , -o Output log to the specified file - (default: file-migration-.log) - --logging-id Identifier for logging - (default: BATCH_RANGE_START) - --logging-interval-ms Interval for logging progres stats - (default: 60000, 1min) - -Batch range options: - --BATCH_RANGE_START Start date for processing - (default: ${args.BATCH_RANGE_START}) - --BATCH_RANGE_END End date for processing - (default: ${args.BATCH_RANGE_END}) - -Concurrency: - --concurrency Number of files to process concurrently - (default: 10) - --concurrent-batches Number of project batches to process concurrently - (default: 1) - --stream-high-water-mark n In-Memory buffering threshold - (default: 1MiB) - -Retries: - --retries Number of times to retry processing a file - (default: 10) - --retry-delay-ms How long to wait before processing a file again - (default: 100, 100ms) - --retry-filestore-404 Retry downloading a file when receiving a 404 - (default: false) - -Other options: - --report Display a report of the current status - --dry-run, -n Perform a dry run without making changes - --help, -h Show this help message - --buffer-dir-prefix

Folder/prefix for buffering files on disk - (default: ${args['buffer-dir-prefix']}) - --sleep-before-exit-ms Defer exiting from the script - (default: 1000, 1s) - -Typical usage: - - node back_fill_file_hash.mjs --all - -is equivalent to - - node back_fill_file_hash.mjs --projects --deleted-projects -`) - process.exit(0) - } - - // Require at least one of --projects, --deleted-projects and --all or --report - if ( - !args.projects && - !args['deleted-projects'] && - !args.all && - !args.report - ) { - console.error( - 'Must specify at least one of --projects and --deleted-projects, --all or --report' - ) - process.exit(1) - } - - // Forbid --all with --projects or --deleted-projects - if (args.all && (args.projects || args['deleted-projects'])) { - console.error('Cannot use --all with --projects or --deleted-projects') - process.exit(1) - } - - // Forbid --all, --projects, --deleted-projects with --report - if (args.report && (args.all || args.projects || args['deleted-projects'])) { - console.error( - 'Cannot use --report with --all, --projects or --deleted-projects' - ) - process.exit(1) - } - - // The --all option processes all projects, including deleted ones - // and checks existing hashed files are present in the blob store. - if (args.all) { - args.projects = true - args['deleted-projects'] = true - } - - const BATCH_RANGE_START = objectIdFromInput(args.BATCH_RANGE_START).toString() - const BATCH_RANGE_END = objectIdFromInput(args.BATCH_RANGE_END).toString() - return { - PROCESS_NON_DELETED_PROJECTS: args.projects, - PROCESS_DELETED_PROJECTS: args['deleted-projects'], - PROCESS_HASHED_FILES: !args['skip-hashed-files'], - PROCESS_BLOBS: !args['skip-existing-blobs'], - DRY_RUN: args['dry-run'], - OUTPUT_FILE: args.report ? '-' : args.output, - BATCH_RANGE_START, - BATCH_RANGE_END, - LOGGING_IDENTIFIER: args['logging-id'] || BATCH_RANGE_START, - LOGGING_INTERVAL: args['logging-interval-ms'], - PROJECT_IDS_FROM: args['from-file'], - DISPLAY_REPORT: args.report, - CONCURRENCY: args.concurrency, - CONCURRENT_BATCHES: args['concurrent-batches'], - STREAM_HIGH_WATER_MARK: args['stream-high-water-mark'], - RETRIES: args.retries, - RETRY_DELAY_MS: args['retry-delay-ms'], - RETRY_FILESTORE_404: args['retry-filestore-404'], - BUFFER_DIR_PREFIX: args['buffer-dir-prefix'], - SLEEP_BEFORE_EXIT: args['sleep-before-exit-ms'], - } -} - -const { - PROCESS_NON_DELETED_PROJECTS, - PROCESS_DELETED_PROJECTS, - PROCESS_BLOBS, - PROCESS_HASHED_FILES, - DRY_RUN, - OUTPUT_FILE, - BATCH_RANGE_START, - BATCH_RANGE_END, - LOGGING_IDENTIFIER, - PROJECT_IDS_FROM, - DISPLAY_REPORT, - CONCURRENCY, - CONCURRENT_BATCHES, - RETRIES, - RETRY_DELAY_MS, - RETRY_FILESTORE_404, - BUFFER_DIR_PREFIX, - STREAM_HIGH_WATER_MARK, - LOGGING_INTERVAL, - SLEEP_BEFORE_EXIT, -} = parseArgs() - -// We need to handle the start and end differently as ids of deleted projects are created at time of deletion. -if (process.env.BATCH_RANGE_START || process.env.BATCH_RANGE_END) { - throw new Error('use --BATCH_RANGE_START and --BATCH_RANGE_END') -} - -const BUFFER_DIR = fs.mkdtempSync(BUFFER_DIR_PREFIX) - -// Log output to a file -if (OUTPUT_FILE !== '-') { - console.warn(`Writing logs into ${OUTPUT_FILE}`) -} -logger.initialize('file-migration', { - streams: [ - { - stream: DISPLAY_REPORT - ? process.stderr - : OUTPUT_FILE === '-' - ? process.stdout - : fs.createWriteStream(OUTPUT_FILE, { flags: 'a' }), - }, - ], -}) - -let lastElapsedTime = 0 -async function displayProgress(options = {}) { - if (OUTPUT_FILE === '-') { - return // skip progress tracking when logging to stdout - } - if (options.completedAll) { - process.stdout.write('\n') - return - } - const elapsedTime = Math.floor((performance.now() - processStart) / 1000) - if (lastElapsedTime === elapsedTime && !options.completedBatch) { - // Avoid spamming the console with the same progress message - return - } - lastElapsedTime = elapsedTime - readline.clearLine(process.stdout, 0) - readline.cursorTo(process.stdout, 0) - process.stdout.write( - `Processed ${STATS.projects} projects, elapsed time ${elapsedTime}s` - ) -} - -/** - * Display the stats for the projects or deletedProjects collections. - * - * @param {number} N - Number of samples to take from the collection. - * @param {string} name - Name of the collection being sampled. - * @param {Collection} collection - MongoDB collection to query. - * @param {Object} query - MongoDB query to filter documents. - * @param {Object} projection - MongoDB projection to select fields. - * @param {number} collectionCount - Total number of documents in the collection. - * @returns {Promise} Resolves when stats have been displayed. - */ -async function getStatsForCollection( - N, - name, - collection, - query, - projection, - collectionCount -) { - const stats = { - projectCount: 0, - projectsWithAllHashes: 0, - fileCount: 0, - fileWithHashCount: 0, - fileMissingInHistoryCount: 0, - } - // Pick a random sample of projects and estimate the number of files without hashes - const result = await collection - .aggregate([ - { $sample: { size: N } }, - { $match: query }, - { - $project: projection, - }, - ]) - .toArray() - - for (const project of result) { - const fileTree = JSON.stringify(project, [ - 'project', - 'rootFolder', - 'folders', - 'fileRefs', - 'hash', - '_id', - ]) - // count the number of files without a hash, these are uniquely identified - // by entries with {"_id":"...."} since we have filtered the file tree - const filesWithoutHash = fileTree.match(/\{"_id":"[0-9a-f]{24}"\}/g) || [] - // count the number of files with a hash, these are uniquely identified - // by the number of "hash" strings due to the filtering - const filesWithHash = fileTree.match(/"hash":"[0-9a-f]{40}"/g) || [] - stats.fileCount += filesWithoutHash.length + filesWithHash.length - stats.fileWithHashCount += filesWithHash.length - stats.projectCount++ - stats.projectsWithAllHashes += filesWithoutHash.length === 0 ? 1 : 0 - const projectId = project._id.toString() - const { blobs: perProjectBlobs } = await getProjectBlobsBatch([projectId]) - const blobs = new Set( - (perProjectBlobs.get(projectId) || []).map(b => b.getHash()) - ) - const uniqueHashes = new Set(filesWithHash.map(m => m.slice(8, 48))) - for (const hash of uniqueHashes) { - if (blobs.has(hash) || GLOBAL_BLOBS.has(hash)) continue - stats.fileMissingInHistoryCount++ - } - } - console.log(`Sampled stats for ${name}:`) - const fractionSampled = stats.projectCount / collectionCount - const percentageSampled = (fractionSampled * 100).toFixed(0) - const fractionConverted = stats.projectsWithAllHashes / stats.projectCount - const fractionToBackFill = 1 - fractionConverted - const percentageToBackFill = (fractionToBackFill * 100).toFixed(0) - const fractionMissing = stats.fileMissingInHistoryCount / stats.fileCount - const percentageMissing = (fractionMissing * 100).toFixed(0) - console.log( - `- Sampled ${name}: ${stats.projectCount} (${percentageSampled}% of all ${name})` - ) - console.log( - `- Sampled ${name} with all hashes present: ${stats.projectsWithAllHashes}` - ) - console.log( - `- Percentage of ${name} that need back-filling hashes: ${percentageToBackFill}% (estimated)` - ) - console.log( - `- Sampled ${name} have ${stats.fileCount} files that need to be checked against the full project history system.` - ) - console.log( - `- Sampled ${name} have ${stats.fileMissingInHistoryCount} files that need to be uploaded to the full project history system (estimating ${percentageMissing}% of all files).` - ) -} - -/** - * Displays a report of the current status of projects and deleted projects, - * including counts and estimated progress based on a sample. - */ -async function displayReport() { - const projectsCountResult = await projectsCollection.estimatedDocumentCount() - const deletedProjectsCountResult = - await deletedProjectsCollection.estimatedDocumentCount() - const sampleSize = 1000 - console.log('Current status:') - console.log(`- Total number of projects: ${projectsCountResult}`) - console.log( - `- Total number of deleted projects: ${deletedProjectsCountResult}` - ) - console.log(`Sampling ${sampleSize} projects to estimate progress...`) - await getStatsForCollection( - sampleSize, - 'projects', - projectsCollection, - { rootFolder: { $exists: true } }, - { rootFolder: 1 }, - projectsCountResult - ) - await getStatsForCollection( - sampleSize, - 'deleted projects', - deletedProjectsCollection, - { 'project.rootFolder': { $exists: true } }, - { 'project.rootFolder': 1 }, - deletedProjectsCountResult - ) -} - -// Filestore endpoint location (configured by /etc/overleaf/env.sh) -const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1' -const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009' - -async function fetchFromFilestore(projectId, fileId) { - const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}` - const response = await fetch(url) - if (!response.ok) { - if (response.status === 404) { - throw new NotFoundError('file not found in filestore', { - status: response.status, - }) - } - const body = await response.text() - throw new OError('fetchFromFilestore failed', { - projectId, - fileId, - status: response.status, - body, - }) - } - if (!response.body) { - throw new OError('fetchFromFilestore response has no body', { - projectId, - fileId, - status: response.status, - }) - } - return response.body -} - -const projectsCollection = db.collection('projects') -/** @type {ProjectsCollection} */ -const typedProjectsCollection = db.collection('projects') -const deletedProjectsCollection = db.collection('deletedProjects') -/** @type {DeletedProjectsCollection} */ -const typedDeletedProjectsCollection = db.collection('deletedProjects') - -const concurrencyLimit = pLimit(CONCURRENCY) - -/** - * @template T - * @template V - * @param {Array} array - * @param {(arg: T) => Promise} fn - * @return {Promise>>} - */ -async function processConcurrently(array, fn) { - return await Promise.all(array.map(x => concurrencyLimit(() => fn(x)))) -} - -const STATS = { - projects: 0, - blobs: 0, - filesWithHash: 0, - filesWithoutHash: 0, - filesDuplicated: 0, - filesRetries: 0, - filesFailed: 0, - fileTreeUpdated: 0, - badFileTrees: 0, - globalBlobsCount: 0, - globalBlobsEgress: 0, - projectDeleted: 0, - projectHardDeleted: 0, - fileHardDeleted: 0, - mongoUpdates: 0, - readFromGCSCount: 0, - readFromGCSIngress: 0, - writeToGCSCount: 0, - writeToGCSEgress: 0, -} - -const processStart = performance.now() -let lastLogTS = processStart -let lastLog = Object.assign({}, STATS) -let lastEventLoopStats = performance.eventLoopUtilization() - -/** - * @param {number} v - * @param {number} ms - */ -function toMiBPerSecond(v, ms) { - const MiB = 1024 * 1024 - return v / MiB / (ms / 1000) -} - -/** - * @param {any} stats - * @param {number} ms - * @return {{readFromGCSThroughputMiBPerSecond: number}} - */ -function bandwidthStats(stats, ms) { - return { - readFromGCSThroughputMiBPerSecond: toMiBPerSecond( - stats.readFromGCSIngress, - ms - ), - } -} - -/** - * @param {EventLoopUtilization} nextEventLoopStats - * @param {number} now - * @return {Object} - */ -function computeDiff(nextEventLoopStats, now) { - const ms = now - lastLogTS - lastLogTS = now - const diff = { - eventLoop: performance.eventLoopUtilization( - nextEventLoopStats, - lastEventLoopStats - ), - } - for (const [name, v] of Object.entries(STATS)) { - diff[name] = v - lastLog[name] - } - return Object.assign(diff, bandwidthStats(diff, ms)) -} - -/** - * @param {boolean} isLast - */ -function printStats(isLast = false) { - const now = performance.now() - const nextEventLoopStats = performance.eventLoopUtilization() - const logLine = { - time: new Date(), - LOGGING_IDENTIFIER, - ...STATS, - ...bandwidthStats(STATS, now - processStart), - eventLoop: nextEventLoopStats, - diff: computeDiff(nextEventLoopStats, now), - deferredBatches: Array.from(deferredBatches.keys()), - } - if (isLast && OUTPUT_FILE === '-') { - console.warn(JSON.stringify(logLine)) - } else { - logger.info(logLine, 'file-migration stats') - } - lastEventLoopStats = nextEventLoopStats - lastLog = Object.assign({}, STATS) -} - -setInterval(printStats, LOGGING_INTERVAL) - -let gracefulShutdownInitiated = false - -process.on('SIGINT', handleSignal) -process.on('SIGTERM', handleSignal) - -function handleSignal() { - gracefulShutdownInitiated = true - console.warn('graceful shutdown initiated, draining queue') -} - -/** - * @param {QueueEntry} entry - * @return {Promise} - */ -async function processFileWithCleanup(entry) { - const { - ctx: { projectId }, - cacheKey, - } = entry - const filePath = Path.join(BUFFER_DIR, projectId.toString() + cacheKey) - try { - return await processFile(entry, filePath) - } finally { - if (!DRY_RUN) { - await fs.promises.rm(filePath, { force: true }) - } - } -} - -/** - * @param {QueueEntry} entry - * @param {string} filePath - * @return {Promise} - */ -async function processFile(entry, filePath) { - for (let attempt = 0; attempt < RETRIES; attempt++) { - try { - return await processFileOnce(entry, filePath) - } catch (err) { - if (gracefulShutdownInitiated) throw err - if (err instanceof NotFoundError) { - if (!RETRY_FILESTORE_404) { - throw err // disable retries for not found in filestore bucket case - } - } - STATS.filesRetries++ - const { - ctx: { projectId }, - fileId, - hash, - path, - } = entry - logger.warn( - { err, projectId, fileId, hash, path, attempt }, - 'failed to process file, trying again' - ) - const jitter = Math.random() * RETRY_DELAY_MS - await setTimeout(RETRY_DELAY_MS + jitter) - } - } - return await processFileOnce(entry, filePath) -} - -/** - * @param {QueueEntry} entry - * @param {string} filePath - * @return {Promise} - */ -async function processFileOnce(entry, filePath) { - const { - ctx: { projectId, historyId }, - fileId, - } = entry - if (entry.hash && entry.ctx.hasCompletedBlob(entry.hash)) { - // We can enter this case for two identical files in the same project, - // one with hash, the other without. When the one without hash gets - // processed first, we can skip downloading the other one we already - // know the hash of. - return entry.hash - } - if (DRY_RUN) { - return // skip processing in dry-run mode by returning undefined - } - const blobStore = new BlobStore(historyId) - STATS.readFromGCSCount++ - // make a fetch request to filestore itself - const src = await fetchFromFilestore(projectId, fileId) - const dst = fs.createWriteStream(filePath, { - highWaterMark: STREAM_HIGH_WATER_MARK, - }) - try { - await Stream.promises.pipeline(src, dst) - } finally { - STATS.readFromGCSIngress += dst.bytesWritten - } - const blob = await makeBlobForFile(filePath) - blob.setStringLength( - await getStringLengthOfFile(blob.getByteLength(), filePath) - ) - const hash = blob.getHash() - if (entry.hash && hash !== entry.hash) { - throw new OError('hash mismatch', { entry, hash }) - } - - if (GLOBAL_BLOBS.has(hash)) { - STATS.globalBlobsCount++ - STATS.globalBlobsEgress += estimateBlobSize(blob) - return hash - } - if (entry.ctx.hasCompletedBlob(hash)) { - return hash - } - entry.ctx.recordPendingBlob(hash) - - try { - await uploadBlobToGCS(blobStore, entry, blob, hash, filePath) - entry.ctx.recordCompletedBlob(hash) // mark upload as completed - } catch (err) { - entry.ctx.recordFailedBlob(hash) - throw err - } - return hash -} - -/** - * @param {BlobStore} blobStore - * @param {QueueEntry} entry - * @param {Blob} blob - * @param {string} hash - * @param {string} filePath - * @return {Promise} - */ -async function uploadBlobToGCS(blobStore, entry, blob, hash, filePath) { - if (entry.ctx.getCachedHistoryBlob(hash)) { - return // fast-path using hint from pre-fetched blobs - } - if (!PROCESS_BLOBS) { - // round trip to postgres/mongo when not pre-fetched - const blob = await blobStore.getBlob(hash) - if (blob) { - entry.ctx.recordHistoryBlob(blob) - return - } - } - // blob missing in history-v1, create in GCS and persist in postgres/mongo - STATS.writeToGCSCount++ - STATS.writeToGCSEgress += blob.getByteLength() - await blobStore.putBlob(filePath, blob) - entry.ctx.recordHistoryBlob(blob) -} - -/** - * @param {Array} files - * @return {Promise} - */ -async function processFiles(files) { - await processConcurrently( - files, - /** - * @param {QueueEntry} entry - * @return {Promise} - */ - async function (entry) { - if (gracefulShutdownInitiated) return - try { - await entry.ctx.processFile(entry) - } catch (err) { - STATS.filesFailed++ - const { - ctx: { projectId }, - fileId, - hash, - path, - } = entry - logger.error( - { err, projectId, fileId, hash, path }, - 'failed to process file' - ) - } - } - ) -} - -/** @type {Map} */ -const deferredBatches = new Map() - -async function waitForDeferredQueues() { - // Wait for ALL pending batches to finish, especially wait for their mongo - // writes to finish to avoid extra work when resuming the batch. - const all = await Promise.allSettled(deferredBatches.values()) - displayProgress({ completedAll: true }) - // Now that all batches finished, we can throw if needed. - for (const res of all) { - if (res.status === 'rejected') { - throw res.reason - } - } -} - -/** - * @param {Array} batch - * @param {string} prefix - */ -async function queueNextBatch(batch, prefix = 'rootFolder.0') { - if (gracefulShutdownInitiated) { - throw new Error('graceful shutdown: aborting batch processing') - } - - // Read ids now, the batch will get trimmed by processBatch shortly. - const start = renderObjectId(batch[0]._id) - const end = renderObjectId(batch[batch.length - 1]._id) - const deferred = processBatch(batch, prefix) - .then(() => { - logger.info({ end }, 'actually completed batch') - displayProgress({ completedBatch: true }) - }) - - .catch(err => { - logger.error({ err, start, end }, 'fatal error processing batch') - throw err - }) - .finally(() => { - deferredBatches.delete(end) - }) - deferredBatches.set(end, deferred) - - if (deferredBatches.size >= CONCURRENT_BATCHES) { - // Wait for any of the deferred batches to finish before fetching the next. - // We should never have more than CONCURRENT_BATCHES batches in memory. - await Promise.race(deferredBatches.values()) - } -} - -/** - * @param {Array} batch - * @param {string} prefix - * @return {Promise} - */ -async function processBatch(batch, prefix = 'rootFolder.0') { - const { nBlobs, blobs } = await collectProjectBlobs(batch) - const files = Array.from(findFileInBatch(batch, prefix, blobs)) - STATS.projects += batch.length - STATS.blobs += nBlobs - - // GC - batch.length = 0 - blobs.clear() - - // The files are currently ordered by project-id. - // Order them by file-id ASC then hash ASC to - // increase the hit rate on the "already processed - // hash for project" checks. - files.sort( - /** - * @param {QueueEntry} a - * @param {QueueEntry} b - * @return {number} - */ - function (a, b) { - if (a.fileId && b.fileId) return a.fileId > b.fileId ? 1 : -1 - if (a.hash && b.hash) return a.hash > b.hash ? 1 : -1 - if (a.fileId) return -1 - return 1 - } - ) - await processFiles(files) - await processConcurrently( - files, - /** - * @param {QueueEntry} entry - * @return {Promise} - */ - async function (entry) { - await entry.ctx.flushMongoQueues() - } - ) -} - -/** - * @param {Array<{project: Project}>} batch - * @return {Promise} - */ -async function handleDeletedFileTreeBatch(batch) { - await queueNextBatch( - batch.map(d => d.project), - 'project.rootFolder.0' - ) -} - -/** - * @param {QueueEntry} entry - * @return {Promise} - */ -async function tryUpdateFileRefInMongo(entry) { - if (DRY_RUN) { - return true // skip mongo updates in dry-run mode - } - if (entry.path.startsWith('project.')) { - return await tryUpdateFileRefInMongoInDeletedProject(entry) - } - - STATS.mongoUpdates++ - const result = await projectsCollection.updateOne( - { - _id: entry.ctx.projectId, - [`${entry.path}._id`]: new ObjectId(entry.fileId), - }, - { - $set: { [`${entry.path}.hash`]: entry.hash }, - } - ) - return result.matchedCount === 1 -} - -/** - * @param {QueueEntry} entry - * @return {Promise} - */ -async function tryUpdateFileRefInMongoInDeletedProject(entry) { - if (DRY_RUN) { - return true // skip mongo updates in dry-run mode - } - STATS.mongoUpdates++ - const result = await deletedProjectsCollection.updateOne( - { - 'deleterData.deletedProjectId': entry.ctx.projectId, - [`${entry.path}._id`]: new ObjectId(entry.fileId), - }, - { - $set: { [`${entry.path}.hash`]: entry.hash }, - } - ) - return result.matchedCount === 1 -} - -const RETRY_UPDATE_HASH = 100 - -/** - * @param {QueueEntry} entry - * @return {Promise} - */ -async function updateFileRefInMongo(entry) { - if (await tryUpdateFileRefInMongo(entry)) return - - const { fileId } = entry - const { projectId } = entry.ctx - for (let i = 0; i < RETRY_UPDATE_HASH; i++) { - let prefix = 'rootFolder.0' - let p = await projectsCollection.findOne( - { _id: projectId }, - { projection: { rootFolder: 1 } } - ) - if (!p) { - STATS.projectDeleted++ - prefix = 'project.rootFolder.0' - const deletedProject = await deletedProjectsCollection.findOne( - { - 'deleterData.deletedProjectId': projectId, - project: { $exists: true }, - }, - { projection: { 'project.rootFolder': 1 } } - ) - p = deletedProject?.project - if (!p) { - STATS.projectHardDeleted++ - console.warn( - 'bug: project hard-deleted while processing', - projectId, - fileId - ) - return - } - } - let found = false - for (const e of findFiles(entry.ctx, p.rootFolder[0], prefix)) { - found = e.fileId === fileId - if (!found) continue - if (await tryUpdateFileRefInMongo(e)) return - break - } - if (!found) { - STATS.fileHardDeleted++ - console.warn('bug: file hard-deleted while processing', projectId, fileId) - return - } - - STATS.fileTreeUpdated++ - } - throw new OError( - 'file-tree updated repeatedly while trying to add hash', - entry - ) -} - -/** - * @param {ProjectContext} ctx - * @param {Folder} folder - * @param {string} path - * @param {boolean} isInputLoop - * @return Generator - */ -function* findFiles(ctx, folder, path, isInputLoop = false) { - if (!folder || typeof folder !== 'object') { - ctx.fileTreeBroken = true - logger.warn({ projectId: ctx.projectId, path }, 'bad file-tree, bad folder') - return - } - if (!Array.isArray(folder.folders)) { - folder.folders = [] - ctx.fileTreeBroken = true - logger.warn( - { projectId: ctx.projectId, path: `${path}.folders` }, - 'bad file-tree, bad folders' - ) - } - let i = 0 - for (const child of folder.folders) { - const idx = i++ - yield* findFiles(ctx, child, `${path}.folders.${idx}`, isInputLoop) - } - if (!Array.isArray(folder.fileRefs)) { - folder.fileRefs = [] - ctx.fileTreeBroken = true - logger.warn( - { projectId: ctx.projectId, path: `${path}.fileRefs` }, - 'bad file-tree, bad fileRefs' - ) - } - i = 0 - for (const fileRef of folder.fileRefs) { - const idx = i++ - const fileRefPath = `${path}.fileRefs.${idx}` - if (!fileRef._id || !(fileRef._id instanceof ObjectId)) { - ctx.fileTreeBroken = true - logger.warn( - { projectId: ctx.projectId, path: fileRefPath }, - 'bad file-tree, bad fileRef id' - ) - continue - } - const fileId = fileRef._id.toString() - if (PROCESS_HASHED_FILES && fileRef.hash) { - if (ctx.canSkipProcessingHashedFile(fileRef.hash)) continue - if (isInputLoop) { - ctx.remainingQueueEntries++ - STATS.filesWithHash++ - } - yield { - ctx, - cacheKey: fileRef.hash, - fileId, - path: MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE, - hash: fileRef.hash, - } - } - if (!fileRef.hash) { - if (isInputLoop) { - ctx.remainingQueueEntries++ - STATS.filesWithoutHash++ - } - yield { - ctx, - cacheKey: fileId, - fileId, - path: fileRefPath, - } - } - } -} - -/** - * @param {Array} projects - * @param {string} prefix - * @param {Map>} blobs - * @return Generator - */ -function* findFileInBatch(projects, prefix, blobs) { - for (const project of projects) { - const projectIdS = project._id.toString() - const historyIdS = project.overleaf.history.id.toString() - const projectBlobs = blobs.get(historyIdS) || [] - const ctx = new ProjectContext(project._id, historyIdS, projectBlobs) - try { - yield* findFiles(ctx, project.rootFolder?.[0], prefix, true) - } catch (err) { - logger.error( - { err, projectId: projectIdS }, - 'bad file-tree, processing error' - ) - } finally { - if (ctx.fileTreeBroken) STATS.badFileTrees++ - } - } -} - -/** - * @param {Array} batch - * @return {Promise<{nBlobs: number, blobs: Map>}>} - */ -async function collectProjectBlobs(batch) { - if (!PROCESS_BLOBS) return { nBlobs: 0, blobs: new Map() } - return await getProjectBlobsBatch(batch.map(p => p.overleaf.history.id)) -} - -const BATCH_FILE_UPDATES = 100 - -const MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE = 'skip-write-to-file-tree' - -class ProjectContext { - /** @type {Map} */ - #historyBlobs - - /** @type {number} */ - remainingQueueEntries = 0 - - /** @type {boolean} */ - fileTreeBroken = false - - /** - * @param {ObjectId} projectId - * @param {string} historyId - * @param {Array} blobs - */ - constructor(projectId, historyId, blobs) { - this.projectId = projectId - this.historyId = historyId - this.#historyBlobs = new Map(blobs.map(b => [b.getHash(), b])) - } - - /** - * @param {string} hash - * @return {Blob | undefined} - */ - getCachedHistoryBlob(hash) { - return this.#historyBlobs.get(hash) - } - - /** - * @param {Blob} blob - */ - recordHistoryBlob(blob) { - this.#historyBlobs.set(blob.getHash(), blob) - } - - /** - * @param {string} hash - * @return {boolean} - */ - canSkipProcessingHashedFile(hash) { - if (this.#historyBlobs.has(hash)) return true // This file will be processed as blob. - if (GLOBAL_BLOBS.has(hash)) return true // global blob - return false - } - - async flushMongoQueuesIfNeeded() { - if (this.remainingQueueEntries === 0) { - await this.flushMongoQueues() - } - - if (this.#pendingFileWrites.length > BATCH_FILE_UPDATES) { - await this.#storeFileHashes() - } - } - - async flushMongoQueues() { - await this.#storeFileHashes() - } - - /** @type {Set} */ - #pendingBlobs = new Set() - /** @type {Set} */ - #completedBlobs = new Set() - - /** - * @param {string} hash - */ - recordPendingBlob(hash) { - this.#pendingBlobs.add(hash) - } - - /** - * @param {string} hash - */ - recordFailedBlob(hash) { - this.#pendingBlobs.delete(hash) - } - - /** - * @param {string} hash - */ - recordCompletedBlob(hash) { - this.#completedBlobs.add(hash) - this.#pendingBlobs.delete(hash) - } - - /** - * @param {string} hash - * @return {boolean} - */ - hasCompletedBlob(hash) { - return this.#pendingBlobs.has(hash) || this.#completedBlobs.has(hash) - } - - /** @type {Array} */ - #pendingFileWrites = [] - - /** - * @param {QueueEntry} entry - */ - queueFileForWritingHash(entry) { - if (entry.path === MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE) return - this.#pendingFileWrites.push(entry) - } - - /** - * @param {Collection} collection - * @param {Array} entries - * @param {Object} query - * @return {Promise>} - */ - async #tryBatchHashWrites(collection, entries, query) { - if (entries.length === 0) return [] - if (DRY_RUN) return [] // skip mongo updates in dry-run mode - const update = {} - for (const entry of entries) { - query[`${entry.path}._id`] = new ObjectId(entry.fileId) - update[`${entry.path}.hash`] = entry.hash - } - STATS.mongoUpdates++ - const result = await collection.updateOne(query, { $set: update }) - if (result.matchedCount === 1) { - return [] // all updated - } - return entries - } - - async #storeFileHashes() { - if (this.#pendingFileWrites.length === 0) return - const individualUpdates = [] - const projectEntries = [] - const deletedProjectEntries = [] - for (const entry of this.#pendingFileWrites) { - if (entry.path.startsWith('project.')) { - deletedProjectEntries.push(entry) - } else { - projectEntries.push(entry) - } - } - this.#pendingFileWrites.length = 0 - - // Try to process them together, otherwise fallback to individual updates and retries. - individualUpdates.push( - ...(await this.#tryBatchHashWrites(projectsCollection, projectEntries, { - _id: this.projectId, - })) - ) - individualUpdates.push( - ...(await this.#tryBatchHashWrites( - deletedProjectsCollection, - deletedProjectEntries, - { 'deleterData.deletedProjectId': this.projectId } - )) - ) - for (const entry of individualUpdates) { - await updateFileRefInMongo(entry) - } - } - - /** @type {Map>} */ - #pendingFiles = new Map() - - /** - * @param {QueueEntry} entry - */ - async processFile(entry) { - if (this.#pendingFiles.has(entry.cacheKey)) { - STATS.filesDuplicated++ - } else { - this.#pendingFiles.set(entry.cacheKey, processFileWithCleanup(entry)) - } - try { - const hash = await this.#pendingFiles.get(entry.cacheKey) - if (!hash) { - if (DRY_RUN) { - return // hash is undefined in dry-run mode - } else { - throw new Error('undefined hash outside dry-run mode') - } - } else { - entry.hash = hash - } - } finally { - this.remainingQueueEntries-- - } - this.queueFileForWritingHash(entry) - await this.flushMongoQueuesIfNeeded() - } -} - -/** - * @param {Blob} blob - * @return {number} - */ -function estimateBlobSize(blob) { - let size = blob.getByteLength() - if (blob.getStringLength()) { - // approximation for gzip (25 bytes gzip overhead and 20% compression ratio) - size = 25 + Math.ceil(size * 0.2) - } - return size -} - -async function processProjectsFromFile() { - const rl = readline.createInterface({ - input: fs.createReadStream(PROJECT_IDS_FROM), - }) - for await (const projectId of rl) { - if (!projectId) continue // skip over trailing new line - let project = await typedProjectsCollection.findOne( - { _id: new ObjectId(projectId) }, - { projection: { rootFolder: 1, _id: 1, 'overleaf.history.id': 1 } } - ) - let prefix = 'rootFolder.0' - if (!project) { - const deletedProject = await typedDeletedProjectsCollection.findOne( - { 'deleterData.deletedProjectId': new ObjectId(projectId) }, - { - projection: { - 'project.rootFolder': 1, - 'project._id': 1, - 'project.overleaf.history.id': 1, - }, - } - ) - if (!deletedProject?.project) { - logger.warn({ projectId }, 'project hard-deleted') - continue - } - project = deletedProject.project - prefix = 'project.rootFolder.0' - } - if (!project?.overleaf?.history?.id) { - logger.warn({ projectId }, 'project has no history id') - continue - } - try { - await queueNextBatch([project], prefix) - } catch (err) { - gracefulShutdownInitiated = true - await waitForDeferredQueues() - throw err - } - } - await waitForDeferredQueues() - console.warn('Done updating projects from input file') -} - -async function processNonDeletedProjects() { - try { - await batchedUpdate( - projectsCollection, - { 'overleaf.history.id': { $exists: true } }, - queueNextBatch, - { rootFolder: 1, _id: 1, 'overleaf.history.id': 1 }, - {}, - { - BATCH_RANGE_START, - BATCH_RANGE_END, - trackProgress: async message => {}, - } - ) - } catch (err) { - gracefulShutdownInitiated = true - throw err - } finally { - await waitForDeferredQueues() - } - console.warn('Done updating live projects') -} - -async function processDeletedProjects() { - try { - await batchedUpdate( - deletedProjectsCollection, - { - 'deleterData.deletedProjectId': { - $gt: new ObjectId(BATCH_RANGE_START), - $lte: new ObjectId(BATCH_RANGE_END), - }, - 'project.overleaf.history.id': { $exists: true }, - }, - handleDeletedFileTreeBatch, - { - 'project.rootFolder': 1, - 'project._id': 1, - 'project.overleaf.history.id': 1, - }, - {}, - { trackProgress: async message => {} } - ) - } catch (err) { - gracefulShutdownInitiated = true - throw err - } finally { - await waitForDeferredQueues() - } - console.warn('Done updating deleted projects') -} - -async function main() { - console.log('Starting project file backup...') - await loadGlobalBlobs() - console.log('Loaded global blobs:', GLOBAL_BLOBS.size) - if (PROJECT_IDS_FROM) { - console.log( - `Processing projects from file: ${PROJECT_IDS_FROM}, this may take a while...` - ) - await processProjectsFromFile() - } else { - if (PROCESS_NON_DELETED_PROJECTS) { - console.log('Processing non-deleted projects...') - await processNonDeletedProjects() - } - if (PROCESS_DELETED_PROJECTS) { - console.log('Processing deleted projects...') - await processDeletedProjects() - } - } - console.warn('Done.') -} - -async function cleanupBufferDir() { - try { - // Perform non-recursive removal of the BUFFER_DIR. Individual files - // should get removed in parallel as part of batch processing. - await fs.promises.rmdir(BUFFER_DIR) - } catch (err) { - console.error(`cleanup of BUFFER_DIR=${BUFFER_DIR} failed`, err) - } -} - -if (DISPLAY_REPORT) { - await cleanupBufferDir() - console.warn('Displaying report...') - await displayReport() - process.exit(0) -} - -try { - try { - await main() - } finally { - printStats(true) - await cleanupBufferDir() - } - - let code = 0 - if (STATS.filesFailed > 0) { - console.warn( - `Some files could not be processed, see logs in ${OUTPUT_FILE} and try again` - ) - code++ - } - if (STATS.fileHardDeleted > 0) { - console.warn( - 'Some hashes could not be updated as the files were hard-deleted, this should not happen' - ) - code++ - } - if (STATS.projectHardDeleted > 0) { - console.warn( - 'Some hashes could not be updated as the project was hard-deleted, this should not happen' - ) - code++ - } - console.warn('-'.repeat(79)) - if (code === 0) { - const allProcessed = - !DRY_RUN && - PROCESS_NON_DELETED_PROJECTS && - PROCESS_DELETED_PROJECTS && - PROCESS_HASHED_FILES && - !PROJECT_IDS_FROM && - usesDefaultBatchRange() - if (allProcessed) { - await db - .collection('migrations') - .updateOne( - { name: '20250519101128_binary_files_migration' }, - { $set: { migratedAt: new Date(DEFAULT_BATCH_RANGE_END_DATE) } }, - { upsert: true } - ) - console.warn('The binary files migration succeeded.') - console.warn( - 'You can now proceed to OVERLEAF_FILESTORE_MIGRATION_LEVEL=2.' - ) - } else { - console.warn( - 'The binary files migration succeeded on a subset of files (at least one of --dry-run, --skip-hashed-files, --from-file, --BATCH_RANGE_START or --BATCH_RANGE_END is set and --all is not set).' - ) - console.warn( - 'Once you are done with all the partial runs, you need to run the migration again on all projects/files to ensure that all files are migrated into the full project history system.' - ) - console.warn('The full run will unlock the upgrade to Server Pro 6.0.') - } - } else { - console.warn('The binary files migration failed, see above.') - console.warn( - 'Please review the failures and check the docs on remediating the failures.' - ) - console.warn( - 'Docs: https://docs.overleaf.com/on-premises/release-notes/release-notes-5.x.x/binary-files-migration#troubleshooting' - ) - console.warn( - 'In case there is not solution available, please reach out to support as detailed in the docs.' - ) - } - console.warn('-'.repeat(79)) - await setTimeout(SLEEP_BEFORE_EXIT) - process.exit(code) -} catch (err) { - console.error(err) - await setTimeout(SLEEP_BEFORE_EXIT) - process.exit(1) -} diff --git a/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs b/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs deleted file mode 100644 index 2525ee1d6e..0000000000 --- a/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs +++ /dev/null @@ -1,629 +0,0 @@ -// @ts-check -import Events from 'node:events' -import fs from 'node:fs' -import Stream from 'node:stream' -import { ObjectId } from 'mongodb' -import logger from '@overleaf/logger' -import OError from '@overleaf/o-error' -import { Blob } from 'overleaf-editor-core' -import { - BlobStore, - getStringLengthOfFile, - makeBlobForFile, -} from '../lib/blob_store/index.js' -import { db } from '../lib/mongodb.js' -import commandLineArgs from 'command-line-args' -import readline from 'node:readline' -import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js' -import { setTimeout } from 'node:timers/promises' - -// Silence warning. -Events.setMaxListeners(20) - -// Enable caching for ObjectId.toString() -ObjectId.cacheHexString = true - -/** - * @typedef {import("mongodb").Collection} Collection - * @typedef {import("mongodb").Collection} ProjectsCollection - * @typedef {import("mongodb").Collection<{project: Project}>} DeletedProjectsCollection - */ - -/** - * @typedef {Object} FileRef - * @property {ObjectId} _id - * @property {string} hash - */ - -/** - * @typedef {Object} Folder - * @property {Array} folders - * @property {Array} fileRefs - */ - -/** - * @typedef {Object} Project - * @property {ObjectId} _id - * @property {Array} rootFolder - * @property {{history: {id: (number|string)}}} overleaf - */ - -/** - * @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_MISSING_HASH: boolean, LOGS: string}} - */ -function parseArgs() { - const args = commandLineArgs([ - { name: 'fixNotFound', type: String, defaultValue: 'true' }, - { name: 'fixHashMismatch', type: String, defaultValue: 'true' }, - { name: 'fixMissingHash', type: String, defaultValue: 'true' }, - { name: 'logs', type: String, defaultValue: '' }, - ]) - /** - * commandLineArgs cannot handle --foo=false, so go the long way - * @param {string} name - * @return {boolean} - */ - function boolVal(name) { - const v = args[name] - if (['true', 'false'].includes(v)) return v === 'true' - throw new Error(`expected "true" or "false" for boolean option ${name}`) - } - return { - FIX_HASH_MISMATCH: boolVal('fixNotFound'), - FIX_NOT_FOUND: boolVal('fixHashMismatch'), - FIX_MISSING_HASH: boolVal('fixMissingHash'), - LOGS: args.logs, - } -} - -const { FIX_HASH_MISMATCH, FIX_NOT_FOUND, FIX_MISSING_HASH, LOGS } = parseArgs() -if (!LOGS) { - throw new Error('--logs parameter missing') -} -const BUFFER_DIR = fs.mkdtempSync( - process.env.BUFFER_DIR_PREFIX || '/tmp/back_fill_file_hash-' -) -const USER_FILES_BUCKET_NAME = process.env.USER_FILES_BUCKET_NAME || '' -if (!USER_FILES_BUCKET_NAME) { - throw new Error('env var USER_FILES_BUCKET_NAME is missing') -} -// https://nodejs.org/api/stream.html#streamgetdefaulthighwatermarkobjectmode -const STREAM_HIGH_WATER_MARK = parseInt( - process.env.STREAM_HIGH_WATER_MARK || (64 * 1024).toString(), - 10 -) -const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10) - -// Filestore endpoint location -const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1' -const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009' - -async function fetchFromFilestore(projectId, fileId) { - const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}` - const response = await fetch(url) - if (!response.ok) { - if (response.status === 404) { - throw new NotFoundError('file not found in filestore', { - status: response.status, - }) - } - const body = await response.text() - throw new OError('fetchFromFilestore failed', { - projectId, - fileId, - status: response.status, - body, - }) - } - if (!response.body) { - throw new OError('fetchFromFilestore response has no body', { - projectId, - fileId, - status: response.status, - }) - } - return response.body -} - -/** @type {ProjectsCollection} */ -const projectsCollection = db.collection('projects') -/** @type {DeletedProjectsCollection} */ -const deletedProjectsCollection = db.collection('deletedProjects') - -let gracefulShutdownInitiated = false - -process.on('SIGINT', handleSignal) -process.on('SIGTERM', handleSignal) - -function handleSignal() { - gracefulShutdownInitiated = true - console.warn('graceful shutdown initiated, draining queue') -} - -class FileDeletedError extends OError {} - -/** @type {Map} */ -const PROJECT_CACHE = new Map() - -/** - * @param {string} projectId - * @return {Promise<{project: Project, projectSoftDeleted: boolean}>} - */ -async function getProject(projectId) { - const cached = PROJECT_CACHE.get(projectId) - if (cached) return cached - - let projectSoftDeleted - let project = await projectsCollection.findOne({ - _id: new ObjectId(projectId), - }) - if (project) { - projectSoftDeleted = false - } else { - const softDeleted = await deletedProjectsCollection.findOne({ - 'deleterData.deletedProjectId': new ObjectId(projectId), - project: { $exists: true }, - }) - if (!softDeleted) { - throw new OError('project hard-deleted') - } - project = softDeleted.project - projectSoftDeleted = true - } - PROJECT_CACHE.set(projectId, { projectSoftDeleted, project }) - return { projectSoftDeleted, project } -} - -/** - * @param {Folder} folder - * @param {string} fileId - * @return {{path: string, fileRef: FileRef, folder: Folder}|null} - */ -function getFileTreePath(folder, fileId) { - if (!folder) return null - let idx = 0 - if (Array.isArray(folder.fileRefs)) { - for (const fileRef of folder.fileRefs) { - if (fileRef?._id.toString() === fileId) { - return { - fileRef, - path: `.fileRefs.${idx}`, - folder, - } - } - idx++ - } - } - idx = 0 - if (Array.isArray(folder.folders)) { - for (const child of folder.folders) { - const match = getFileTreePath(child, fileId) - if (match) { - return { - fileRef: match.fileRef, - folder: match.folder, - path: `.folders.${idx}${match.path}`, - } - } - idx++ - } - } - return null -} - -/** - * @param {string} projectId - * @param {string} fileId - * @return {Promise<{fileRef: FileRef, folder: Folder, fullPath: string, query: Object, projectSoftDeleted: boolean}>} - */ -async function findFile(projectId, fileId) { - const { projectSoftDeleted, project } = await getProject(projectId) - const match = getFileTreePath(project.rootFolder[0], fileId) - if (!match) { - throw new FileDeletedError('file not found in file-tree', { - projectSoftDeleted, - }) - } - const { path, fileRef, folder } = match - let fullPath - let query - if (projectSoftDeleted) { - fullPath = `project.rootFolder.0${path}` - query = { - 'deleterData.deletedProjectId': new ObjectId(projectId), - [`${fullPath}._id`]: new ObjectId(fileId), - } - } else { - fullPath = `rootFolder.0${path}` - query = { - _id: new ObjectId(projectId), - [`${fullPath}._id`]: new ObjectId(fileId), - } - } - return { - projectSoftDeleted, - query, - fullPath, - fileRef, - folder, - } -} - -/** - * @param {string} line - * @return {Promise} - */ -async function fixNotFound(line) { - const { projectId, fileId, bucketName } = JSON.parse(line) - if (bucketName !== USER_FILES_BUCKET_NAME) { - throw new OError('not found case for another bucket') - } - - const { projectSoftDeleted, query, fullPath, fileRef, folder } = - await findFile(projectId, fileId) - logger.info({ projectId, fileId, fileRef }, 'removing fileRef') - // Copied from _removeElementFromMongoArray (https://github.com/overleaf/internal/blob/11e09528c153de6b7766d18c3c90d94962190371/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js) - const nonArrayPath = fullPath.slice(0, fullPath.lastIndexOf('.')) - let result - if (projectSoftDeleted) { - result = await deletedProjectsCollection.updateOne(query, { - $pull: { [nonArrayPath]: { _id: new ObjectId(fileId) } }, - $inc: { 'project.version': 1 }, - }) - } else { - result = await projectsCollection.updateOne(query, { - $pull: { [nonArrayPath]: { _id: new ObjectId(fileId) } }, - $inc: { version: 1 }, - }) - } - if (result.matchedCount !== 1) { - throw new OError('file-tree write did not match', { result }) - } - // Update the cache. The mongo-path of the next file will be off otherwise. - folder.fileRefs = folder.fileRefs.filter(f => !f._id.equals(fileId)) - return true -} - -/** - * @param {string} projectId - * @param {string} fileId - * @param {string} hash - * @return {Promise} - */ -async function setHashInMongo(projectId, fileId, hash) { - const { projectSoftDeleted, query, fullPath, fileRef } = await findFile( - projectId, - fileId - ) - if (fileRef.hash === hash) return - logger.info({ projectId, fileId, fileRef, hash }, 'setting fileRef hash') - let result - if (projectSoftDeleted) { - result = await deletedProjectsCollection.updateOne(query, { - $set: { [`${fullPath}.hash`]: hash }, - $inc: { 'project.version': 1 }, - }) - } else { - result = await projectsCollection.updateOne(query, { - $set: { [`${fullPath}.hash`]: hash }, - $inc: { version: 1 }, - }) - } - if (result.matchedCount !== 1) { - throw new OError('file-tree write did not match', { result }) - } - fileRef.hash = hash // Update cache for completeness. -} - -/** - * @param {string} projectId - * @param {string} fileId - * @param {string} historyId - * @return {Promise} - */ -async function importRestoredFilestoreFile(projectId, fileId, historyId) { - const path = `${BUFFER_DIR}/${projectId}_${fileId}` - try { - let s - try { - s = await fetchFromFilestore(projectId, fileId) - } catch (err) { - if (err instanceof NotFoundError) { - throw new OError('missing blob, need to restore filestore file', { - projectId, - fileId, - }) - } - throw err - } - await Stream.promises.pipeline( - s, - fs.createWriteStream(path, { highWaterMark: STREAM_HIGH_WATER_MARK }) - ) - const blobStore = new BlobStore(historyId) - const blob = await blobStore.putFile(path) - await setHashInMongo(projectId, fileId, blob.getHash()) - } finally { - await fs.promises.rm(path, { force: true }) - } -} - -/** - * @param {string} projectId - * @param {string} fileId - * @param {string} path - * @return {Promise} - */ -async function bufferFilestoreFileToDisk(projectId, fileId, path) { - try { - await Stream.promises.pipeline( - await fetchFromFilestore(projectId, fileId), - fs.createWriteStream(path, { highWaterMark: STREAM_HIGH_WATER_MARK }) - ) - const blob = await makeBlobForFile(path) - blob.setStringLength( - await getStringLengthOfFile(blob.getByteLength(), path) - ) - return blob - } catch (err) { - if (err instanceof NotFoundError) { - throw new OError('missing blob, need to restore filestore file', { - projectId, - fileId, - }) - } - throw err - } -} - -/** - * @param {string} projectId - * @param {string} fileId - * @return {Promise} - */ -async function computeFilestoreFileHash(projectId, fileId) { - const path = `${BUFFER_DIR}/${projectId}_${fileId}` - try { - const blob = await bufferFilestoreFileToDisk(projectId, fileId, path) - return blob.getHash() - } finally { - await fs.promises.rm(path, { force: true }) - } -} - -/** - * @param {string} projectId - * @param {string} fileId - * @return {Promise} - */ -async function uploadFilestoreFile(projectId, fileId) { - const path = `${BUFFER_DIR}/${projectId}_${fileId}` - try { - const blob = await bufferFilestoreFileToDisk(projectId, fileId, path) - const hash = blob.getHash() - try { - await ensureBlobExistsForFile(projectId, fileId, hash) - } catch (err) { - if (!(err instanceof Blob.NotFoundError)) throw err - - const { project } = await getProject(projectId) - const historyId = project.overleaf.history.id.toString() - const blobStore = new BlobStore(historyId) - await blobStore.putBlob(path, blob) - await ensureBlobExistsForFile(projectId, fileId, hash) - } - } finally { - await fs.promises.rm(path, { force: true }) - } -} - -/** - * @param {string} line - * @return {Promise} - */ -async function fixHashMismatch(line) { - const { - projectId, - fileId, - hash: computedHash, - entry: { - hash: fileTreeHash, - ctx: { historyId }, - }, - } = JSON.parse(line) - const blobStore = new BlobStore(historyId) - if (await blobStore.getBlob(fileTreeHash)) { - throw new OError('found blob with computed filestore object hash') - } - if (!(await blobStore.getBlob(computedHash))) { - await importRestoredFilestoreFile(projectId, fileId, historyId) - return true - } - return await ensureBlobExistsForFile(projectId, fileId, computedHash) -} - -/** - * @param {string} projectId - * @param {string} fileId - * @param {string} hash - * @return {Promise} - */ -async function hashAlreadyUpdatedInFileTree(projectId, fileId, hash) { - const { fileRef } = await findFile(projectId, fileId) - return fileRef.hash === hash -} - -/** - * @param {string} projectId - * @param {string} fileId - * @param {string} hash - * @return {Promise} - */ -async function ensureBlobExistsForFile(projectId, fileId, hash) { - const { project } = await getProject(projectId) - const historyId = project.overleaf.history.id.toString() - const blobStore = new BlobStore(historyId) - if ( - (await hashAlreadyUpdatedInFileTree(projectId, fileId, hash)) && - (await blobStore.getBlob(hash)) - ) { - return false // already processed - } - - const stream = await blobStore.getStream(hash) - const path = `${BUFFER_DIR}/${historyId}_${hash}` - try { - await Stream.promises.pipeline( - stream, - fs.createWriteStream(path, { - highWaterMark: STREAM_HIGH_WATER_MARK, - }) - ) - - const writtenBlob = await makeBlobForFile(path) - writtenBlob.setStringLength( - await getStringLengthOfFile(writtenBlob.getByteLength(), path) - ) - if (writtenBlob.getHash() !== hash) { - // Double check download, better safe than sorry. - throw new OError('blob corrupted', { writtenBlob, hash }) - } - - let blob = await blobStore.getBlob(hash) - if (!blob) { - // Calling blobStore.putBlob would result in the same error again. - // HACK: Skip upload to GCS and finalize putBlob operation directly. - await blobStore.backend.insertBlob(historyId, writtenBlob) - } - } finally { - await fs.promises.rm(path, { force: true }) - } - await setHashInMongo(projectId, fileId, hash) - return true -} - -/** - * @param {string} line - * @return {Promise} - */ -async function fixMissingHash(line) { - let { projectId, _id: fileId } = JSON.parse(line) - const { - fileRef: { hash }, - } = await findFile(projectId, fileId) - if (hash) { - // processed, double check - return await ensureBlobExistsForFile(projectId, fileId, hash) - } - await uploadFilestoreFile(projectId, fileId) - return true -} - -const CASES = { - 'not found': { - match: 'NotFoundError', - flag: FIX_NOT_FOUND, - action: fixNotFound, - }, - 'hash mismatch': { - match: 'OError: hash mismatch', - flag: FIX_HASH_MISMATCH, - action: fixHashMismatch, - }, - 'missing file hash': { - match: '"bad file hash"', - flag: FIX_MISSING_HASH, - action: fixMissingHash, - }, -} - -const STATS = { - processedLines: 0, - success: 0, - alreadyProcessed: 0, - fileDeleted: 0, - skipped: 0, - failed: 0, - unmatched: 0, -} -function logStats() { - console.log( - JSON.stringify({ - time: new Date(), - gracefulShutdownInitiated, - ...STATS, - }) - ) -} -setInterval(logStats, 10_000) - -async function processLog() { - const rl = readline.createInterface({ - input: fs.createReadStream(LOGS), - }) - nextLine: for await (const line of rl) { - if (gracefulShutdownInitiated) break - STATS.processedLines++ - if ( - !( - line.includes('"failed to process file"') || - // Process missing hashes as flagged by find_malformed_filetrees.mjs - line.includes('"bad file-tree path"') - ) - ) { - continue - } - - for (const [name, { match, flag, action }] of Object.entries(CASES)) { - if (!line.includes(match)) continue - if (flag) { - try { - if (await action(line)) { - STATS.success++ - } else { - STATS.alreadyProcessed++ - } - } catch (err) { - if (err instanceof FileDeletedError) { - STATS.fileDeleted++ - logger.info({ err, line }, 'file deleted, skipping') - } else { - STATS.failed++ - logger.error({ err, line }, `failed to fix ${name}`) - } - } - } else { - STATS.skipped++ - } - continue nextLine - } - STATS.unmatched++ - logger.warn({ line }, 'unknown fatal error') - } -} - -async function main() { - try { - await processLog() - } finally { - logStats() - try { - await fs.promises.rm(BUFFER_DIR, { recursive: true, force: true }) - } catch (err) { - console.error(`Cleanup of BUFFER_DIR=${BUFFER_DIR} failed`, err) - } - } - const { skipped, failed, unmatched } = STATS - await setTimeout(SLEEP_BEFORE_EXIT) - if (failed > 0) { - process.exit(Math.min(failed, 99)) - } else if (unmatched > 0) { - process.exit(100) - } else if (skipped > 0) { - process.exit(101) - } else { - process.exit(0) - } -} - -await main() diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs deleted file mode 100644 index 5757fc511e..0000000000 --- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +++ /dev/null @@ -1,1403 +0,0 @@ -import fs from 'node:fs' -import Crypto from 'node:crypto' -import Stream from 'node:stream' -import { setTimeout } from 'node:timers/promises' -import { promisify } from 'node:util' -import { ObjectId, Binary } from 'mongodb' -import { db, globalBlobs } from '../../../../storage/lib/mongodb.js' -import cleanup from './support/cleanup.js' -import testProjects from '../api/support/test_projects.js' -import { execFile } from 'node:child_process' -import chai, { expect } from 'chai' -import chaiExclude from 'chai-exclude' -import { WritableBuffer } from '@overleaf/stream-utils' -import { - backupPersistor, - projectBlobsBucket, -} from '../../../../storage/lib/backupPersistor.mjs' -import { - BlobStore, - makeProjectKey, -} from '../../../../storage/lib/blob_store/index.js' - -import { mockFilestore } from './support/MockFilestore.mjs' - -chai.use(chaiExclude) -const TIMEOUT = 20 * 1_000 - -const projectsCollection = db.collection('projects') -const deletedProjectsCollection = db.collection('deletedProjects') - -/** - * @param {ObjectId} objectId - * @return {string} - */ -function gitBlobHash(objectId) { - return gitBlobHashBuffer(Buffer.from(objectId.toString())) -} - -/** - * @param {Buffer} buf - * @return {string} - */ -function gitBlobHashBuffer(buf) { - const sha = Crypto.createHash('sha1') - sha.update(`blob ${buf.byteLength}\x00`) - sha.update(buf) - return sha.digest('hex') -} - -/** - * @param {string} gitBlobHash - * @return {Binary} - */ -function binaryForGitBlobHash(gitBlobHash) { - return new Binary(Buffer.from(gitBlobHash, 'hex')) -} - -function objectIdFromTime(timestamp) { - return ObjectId.createFromTime(new Date(timestamp).getTime() / 1000) -} - -const PRINT_IDS_AND_HASHES_FOR_DEBUGGING = false - -describe('back_fill_file_hash script', function () { - this.timeout(TIMEOUT) - - const projectId0 = objectIdFromTime('2017-01-01T00:00:00Z') - const projectId1 = objectIdFromTime('2017-01-01T00:01:00Z') - const projectId2 = objectIdFromTime('2017-01-01T00:02:00Z') - const projectId3 = objectIdFromTime('2024-01-01T00:03:00Z') - const projectIdDeleted0 = objectIdFromTime('2017-01-01T00:04:00Z') - const projectIdDeleted1 = objectIdFromTime('2024-01-01T00:05:00Z') - const projectIdNoHistory = objectIdFromTime('2017-01-01T00:06:00Z') - const projectIdNoHistoryDeleted = objectIdFromTime('2017-01-01T00:07:00Z') - const projectIdHardDeleted = objectIdFromTime('2017-01-01T00:08:00Z') - const projectIdNoOverleaf = objectIdFromTime('2017-01-01T00:09:00Z') - const projectIdNoOverleafDeleted = objectIdFromTime('2017-01-01T00:10:00Z') - const projectIdBadFileTree0 = objectIdFromTime('2024-01-01T00:11:00Z') - const projectIdBadFileTree1 = objectIdFromTime('2024-01-01T00:12:00Z') - const projectIdBadFileTree2 = objectIdFromTime('2024-01-01T00:13:00Z') - const projectIdBadFileTree3 = objectIdFromTime('2024-01-01T00:14:00Z') - const historyId0 = 42 // stored as number is mongo - const historyId1 = projectId1.toString() - const historyId2 = projectId2.toString() - const historyId3 = projectId3.toString() - const historyIdDeleted0 = projectIdDeleted0.toString() - const historyIdDeleted1 = projectIdDeleted1.toString() - const historyIdBadFileTree0 = projectIdBadFileTree0.toString() - const historyIdBadFileTree1 = projectIdBadFileTree1.toString() - const historyIdBadFileTree2 = projectIdBadFileTree2.toString() - const historyIdBadFileTree3 = projectIdBadFileTree3.toString() - const fileId0 = objectIdFromTime('2017-02-01T00:00:00Z') - const fileId1 = objectIdFromTime('2017-02-01T00:01:00Z') - const fileId2 = objectIdFromTime('2017-02-01T00:02:00Z') - const fileId3 = objectIdFromTime('2017-02-01T00:03:00Z') - const fileId4 = objectIdFromTime('2017-02-01T00:04:00Z') - const fileId5 = objectIdFromTime('2024-02-01T00:05:00Z') - const fileId6 = objectIdFromTime('2017-02-01T00:06:00Z') - const fileId7 = objectIdFromTime('2017-02-01T00:07:00Z') - const fileId8 = objectIdFromTime('2017-02-01T00:08:00Z') - const fileId9 = objectIdFromTime('2017-02-01T00:09:00Z') - const fileId10 = objectIdFromTime('2017-02-01T00:10:00Z') - const fileId11 = objectIdFromTime('2017-02-01T00:11:00Z') - const contentTextBlob0 = Buffer.from('Hello 0') - const hashTextBlob0 = gitBlobHashBuffer(contentTextBlob0) - const contentTextBlob1 = Buffer.from('Hello 1') - const hashTextBlob1 = gitBlobHashBuffer(contentTextBlob1) - const contentTextBlob2 = Buffer.from('Hello 2') - const hashTextBlob2 = gitBlobHashBuffer(contentTextBlob2) - const contentTextBlob3 = Buffer.from('Hello 3') - const hashTextBlob3 = gitBlobHashBuffer(contentTextBlob3) - const deleteProjectsRecordId0 = new ObjectId() - const deleteProjectsRecordId1 = new ObjectId() - const deleteProjectsRecordId2 = new ObjectId() - const deleteProjectsRecordId3 = new ObjectId() - const deleteProjectsRecordId4 = new ObjectId() - const twoByteUTF8Symbol = 'รถ' - const contentFile7 = Buffer.alloc(4_000_000, twoByteUTF8Symbol) - const hashFile7 = gitBlobHashBuffer(contentFile7) - const potentiallyWrittenBlobs = [ - { projectId: projectId0, historyId: historyId0, fileId: fileId0 }, - // { historyId: projectId0, fileId: fileId6 }, // global blob - { - projectId: projectId0, - historyId: historyId0, - fileId: fileId7, - hash: hashFile7, - content: contentFile7, - }, - { - projectId: projectId0, - historyId: historyId0, - hash: hashTextBlob0, - content: contentTextBlob0, - }, - { - projectId: projectId1, - historyId: historyId1, - hash: hashTextBlob1, - content: contentTextBlob1, - }, - { - projectId: projectId2, - historyId: historyId2, - hash: hashTextBlob2, - content: contentTextBlob2, - }, - { projectId: projectId1, historyId: historyId1, fileId: fileId1 }, - { - projectId: projectId2, - historyId: historyId2, - fileId: fileId2, - hasHash: true, - }, - { projectId: projectId3, historyId: historyId3, fileId: fileId3 }, - // fileId10 is dupe of fileId3, without a hash - { - projectId: projectId3, - historyId: historyId3, - fileId: fileId10, - content: Buffer.from(fileId3.toString()), - hash: gitBlobHash(fileId3), - }, - // fileId11 is dupe of fileId3, but with a hash - { - projectId: projectId3, - historyId: historyId3, - fileId: fileId11, - content: Buffer.from(fileId3.toString()), - hash: gitBlobHash(fileId3), - hasHash: true, - }, - { - projectId: projectIdDeleted0, - historyId: historyIdDeleted0, - fileId: fileId4, - }, - { - projectId: projectIdDeleted1, - historyId: historyIdDeleted1, - fileId: fileId5, - hasHash: true, - }, - { - projectId: projectIdBadFileTree0, - historyId: historyIdBadFileTree0, - hash: hashTextBlob3, - content: contentTextBlob3, - }, - { - projectId: projectIdBadFileTree3, - historyId: historyIdBadFileTree3, - fileId: fileId9, - }, - ] - if (PRINT_IDS_AND_HASHES_FOR_DEBUGGING) { - const fileIds = { - fileId0, - fileId1, - fileId2, - fileId3, - fileId4, - fileId5, - fileId6, - } - console.log({ - projectId0, - projectId1, - projectId2, - projectId3, - projectIdDeleted0, - projectIdDeleted1, - historyId0, - historyId1, - historyId2, - historyId3, - historyIdDeleted0, - historyIdDeleted1, - ...fileIds, - }) - for (const [name, v] of Object.entries(fileIds)) { - console.log( - name, - gitBlobHash(v), - Array.from(binaryForGitBlobHash(gitBlobHash(v)).value()) - ) - } - } - - async function populateMongo() { - await globalBlobs.insertMany([ - { _id: gitBlobHash(fileId6), byteLength: 24, stringLength: 24 }, - { _id: gitBlobHash(fileId8), byteLength: 24, stringLength: 24 }, - ]) - await projectsCollection.insertMany([ - { - _id: projectId0, - rootFolder: [ - { - fileRefs: [ - { _id: fileId8, hash: gitBlobHash(fileId8) }, - { _id: fileId0 }, - { _id: fileId6 }, - { _id: fileId7 }, - ], - folders: [{ fileRefs: [], folders: [] }], - }, - ], - overleaf: { history: { id: historyId0 } }, - }, - { - _id: projectId1, - rootFolder: [ - { - fileRefs: [{ _id: fileId1 }], - folders: [ - { - fileRefs: [], - folders: [{ fileRefs: [{ _id: fileId1 }], folders: [] }], - }, - ], - }, - ], - overleaf: { history: { id: historyId1 } }, - }, - { - _id: projectId2, - rootFolder: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [{ _id: fileId2, hash: gitBlobHash(fileId2) }], - folders: [], - }, - ], - }, - ], - }, - ], - overleaf: { history: { id: historyId2 } }, - }, - { - _id: projectId3, - rootFolder: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [ - { _id: fileId3 }, - { _id: fileId10 }, - { _id: fileId11, hash: gitBlobHash(fileId3) }, - ], - folders: [], - }, - ], - }, - ], - }, - ], - overleaf: { history: { id: historyId3 } }, - }, - { - _id: projectIdNoHistory, - rootFolder: [{ fileRefs: [], folders: [] }], - overleaf: { history: { conversionFailed: true } }, - }, - { - _id: projectIdNoOverleaf, - rootFolder: [{ fileRefs: [], folders: [] }], - }, - { - _id: projectIdBadFileTree0, - overleaf: { history: { id: historyIdBadFileTree0 } }, - }, - { - _id: projectIdBadFileTree1, - rootFolder: [], - overleaf: { history: { id: historyIdBadFileTree1 } }, - }, - { - _id: projectIdBadFileTree2, - rootFolder: [{ fileRefs: [{ _id: null }] }], - overleaf: { history: { id: historyIdBadFileTree2 } }, - }, - { - _id: projectIdBadFileTree3, - rootFolder: [ - { - folders: [null, { folders: {}, fileRefs: 13 }], - fileRefs: [{ _id: fileId9 }], - }, - ], - overleaf: { history: { id: historyIdBadFileTree3 } }, - }, - ]) - await deletedProjectsCollection.insertMany([ - { - _id: deleteProjectsRecordId0, - project: { - _id: projectIdDeleted0, - rootFolder: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [], - folders: [{ fileRefs: [{ _id: fileId4 }], folders: [] }], - }, - ], - }, - ], - overleaf: { history: { id: historyIdDeleted0 } }, - }, - deleterData: { - deletedProjectId: projectIdDeleted0, - }, - }, - { - _id: deleteProjectsRecordId1, - project: { - _id: projectIdDeleted1, - rootFolder: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [{ _id: fileId5, hash: gitBlobHash(fileId5) }], - folders: [], - }, - ], - }, - ], - }, - ], - overleaf: { history: { id: historyIdDeleted1 } }, - }, - deleterData: { - deletedProjectId: projectIdDeleted1, - }, - }, - { - _id: deleteProjectsRecordId2, - project: { - _id: projectIdNoHistoryDeleted, - rootFolder: [{ fileRefs: [], folders: [] }], - overleaf: { history: { conversionFailed: true } }, - }, - deleterData: { - deletedProjectId: projectIdNoHistoryDeleted, - }, - }, - { - _id: deleteProjectsRecordId3, - deleterData: { deletedProjectId: projectIdHardDeleted }, - }, - { - _id: deleteProjectsRecordId4, - project: { - _id: projectIdNoOverleafDeleted, - rootFolder: [{ fileRefs: [], folders: [] }], - }, - deleterData: { - deletedProjectId: projectIdNoOverleafDeleted, - }, - }, - ]) - } - - async function populateHistoryV1() { - await Promise.all([ - testProjects.createEmptyProject(historyId0.toString()), - testProjects.createEmptyProject(historyId1), - testProjects.createEmptyProject(historyId2), - testProjects.createEmptyProject(historyId3), - testProjects.createEmptyProject(historyIdDeleted0), - testProjects.createEmptyProject(historyIdDeleted1), - testProjects.createEmptyProject(historyIdBadFileTree0), - testProjects.createEmptyProject(historyIdBadFileTree1), - testProjects.createEmptyProject(historyIdBadFileTree2), - testProjects.createEmptyProject(historyIdBadFileTree3), - ]) - - const blobStore0 = new BlobStore(historyId0.toString()) - await blobStore0.putString(contentTextBlob0.toString()) - const blobStore1 = new BlobStore(historyId1.toString()) - await blobStore1.putString(contentTextBlob1.toString()) - const blobStore2 = new BlobStore(historyId2.toString()) - await blobStore2.putString(contentTextBlob2.toString()) - const blobStoreBadFileTree = new BlobStore(historyIdBadFileTree0.toString()) - await blobStoreBadFileTree.putString(contentTextBlob3.toString()) - } - - async function populateFilestore() { - await mockFilestore.addFile(projectId0, fileId0, fileId0.toString()) - await mockFilestore.addFile(projectId0, fileId6, fileId6.toString()) - await mockFilestore.addFile(projectId0, fileId7, contentFile7) - await mockFilestore.addFile(projectId1, fileId1, fileId1.toString()) - await mockFilestore.addFile(projectId2, fileId2, fileId2.toString()) - await mockFilestore.addFile(projectId3, fileId3, fileId3.toString()) - await mockFilestore.addFile( - projectId3, - fileId10, - // fileId10 is dupe of fileId3 - fileId3.toString() - ) - await mockFilestore.addFile( - projectId3, - fileId11, - // fileId11 is dupe of fileId3 - fileId3.toString() - ) - await mockFilestore.addFile(projectIdDeleted0, fileId4, fileId4.toString()) - await mockFilestore.addFile(projectIdDeleted1, fileId5, fileId5.toString()) - await mockFilestore.addFile( - projectIdBadFileTree3, - fileId9, - fileId9.toString() - ) - } - - async function prepareEnvironment() { - await cleanup.everything() - // Manually remove the guard that is created after running the binary files migration. - await db - .collection('migrations') - .deleteOne({ name: '20250519101128_binary_files_migration' }) - await mockFilestore.start() - await populateMongo() - await populateHistoryV1() - await populateFilestore() - } - - /** - * @param {Array} args - * @return {Promise<{result: { stdout: string, stderr: string, status: number }, stats: any}>} - */ - async function rawRunScript(args = [], env = {}) { - let result - try { - result = await promisify(execFile)( - process.argv0, - [ - 'storage/scripts/back_fill_file_hash.mjs', - '--sleep-before-exit-ms=1', - ...args, - ], - { - encoding: 'utf-8', - timeout: TIMEOUT - 500, - env: { - ...process.env, - AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE: '1', - LOG_LEVEL: 'warn', // Override LOG_LEVEL of acceptance tests - ...env, - }, - } - ) - result.status = 0 - } catch (err) { - const { stdout, stderr, code } = err - if (typeof code !== 'number') { - console.log(err) - } - result = { stdout, stderr, status: code } - } - // Ensure no tmp folder is left behind. - expect((await fs.promises.readdir('/tmp')).join(';')).to.not.match( - /back_fill_file_hash/ - ) - return result - } - - /** - * @param {Array} args - * @param {boolean} shouldHaveWritten - * @return {Promise<{result, stats: any, migrationCreated: boolean}>} - */ - async function tryRunScript(args = [], shouldHaveWritten) { - const result = await rawRunScript([ - '--output=-', - '--projects', - '--deleted-projects', - ...args, - ]) - const extraStatsKeys = ['eventLoop', 'readFromGCSThroughputMiBPerSecond'] - const stats = JSON.parse( - result.stderr - .split('\n') - .filter(l => l.includes('LOGGING_IDENTIFIER')) - .pop() - ) - expect(Object.keys(stats.diff).sort()).to.deep.equal( - [...extraStatsKeys, ...Object.keys(STATS_ALL)].sort() - ) - delete stats.diff - expect(new Date(stats.time).toISOString()).to.equal(stats.time) - delete stats.time - if (shouldHaveWritten) { - expect(stats.readFromGCSThroughputMiBPerSecond).to.be.greaterThan(0) - } - for (const key of extraStatsKeys) { - delete stats[key] - } - delete stats.LOGGING_IDENTIFIER - expect(stats.deferredBatches).to.have.length( - 0, - 'should not have any remaining deferred batches' - ) - delete stats.deferredBatches - const migrationCreated = !!(await db - .collection('migrations') - .findOne({ name: '20250519101128_binary_files_migration' })) - return { stats, result, migrationCreated } - } - - /** - * @param {Array} args - * @param {boolean} shouldHaveWritten - * @return {Promise<{result, stats: any}>} - */ - async function runScript(args = [], shouldHaveWritten = true) { - const { stats, result, migrationCreated } = await tryRunScript( - args, - shouldHaveWritten - ) - if (result.status !== 0) { - console.log(result) - expect(result).to.have.property('status', 0) - } - return { stats, result, migrationCreated } - } - - /** - * @param {boolean} processHashedFiles - */ - function commonAssertions(processHashedFiles = false) { - const writtenBlobs = potentiallyWrittenBlobs.filter(({ hasHash }) => { - if (processHashedFiles) return true // all files processed - return !hasHash // only files without hash processed - }) - it('should update mongo', async function () { - expect(await projectsCollection.find({}).toArray()) - .excludingEvery([ - 'currentEndTimestamp', - 'currentEndVersion', - 'updatedAt', - 'backup', - ]) - .to.deep.equal([ - { - _id: projectId0, - rootFolder: [ - { - fileRefs: [ - { _id: fileId8, hash: gitBlobHash(fileId8) }, - { _id: fileId0, hash: gitBlobHash(fileId0) }, - { _id: fileId6, hash: gitBlobHash(fileId6) }, - { _id: fileId7, hash: hashFile7 }, - ], - folders: [{ fileRefs: [], folders: [] }], - }, - ], - overleaf: { history: { id: historyId0 } }, - }, - { - _id: projectId1, - rootFolder: [ - { - fileRefs: [{ _id: fileId1, hash: gitBlobHash(fileId1) }], - folders: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [ - { _id: fileId1, hash: gitBlobHash(fileId1) }, - ], - folders: [], - }, - ], - }, - ], - }, - ], - overleaf: { history: { id: historyId1 } }, - }, - { - _id: projectId2, - rootFolder: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [ - { _id: fileId2, hash: gitBlobHash(fileId2) }, - ], - folders: [], - }, - ], - }, - ], - }, - ], - overleaf: { history: { id: historyId2 } }, - }, - { - _id: projectId3, - rootFolder: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [ - { _id: fileId3, hash: gitBlobHash(fileId3) }, - { _id: fileId10, hash: gitBlobHash(fileId3) }, - { _id: fileId11, hash: gitBlobHash(fileId3) }, - ], - folders: [], - }, - ], - }, - ], - }, - ], - overleaf: { history: { id: historyId3 } }, - }, - { - _id: projectIdNoHistory, - rootFolder: [{ fileRefs: [], folders: [] }], - overleaf: { history: { conversionFailed: true } }, - }, - { - _id: projectIdNoOverleaf, - rootFolder: [{ fileRefs: [], folders: [] }], - }, - { - _id: projectIdBadFileTree0, - overleaf: { history: { id: historyIdBadFileTree0 } }, - }, - { - _id: projectIdBadFileTree1, - rootFolder: [], - overleaf: { history: { id: historyIdBadFileTree1 } }, - }, - { - _id: projectIdBadFileTree2, - rootFolder: [{ fileRefs: [{ _id: null }] }], - overleaf: { history: { id: historyIdBadFileTree2 } }, - }, - { - _id: projectIdBadFileTree3, - rootFolder: [ - { - folders: [null, { folders: {}, fileRefs: 13 }], - fileRefs: [{ _id: fileId9, hash: gitBlobHash(fileId9) }], - }, - ], - overleaf: { history: { id: historyIdBadFileTree3 } }, - }, - ]) - expect(await deletedProjectsCollection.find({}).toArray()).to.deep.equal([ - { - _id: deleteProjectsRecordId0, - project: { - _id: projectIdDeleted0, - rootFolder: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [ - { _id: fileId4, hash: gitBlobHash(fileId4) }, - ], - folders: [], - }, - ], - }, - ], - }, - ], - overleaf: { history: { id: historyIdDeleted0 } }, - }, - deleterData: { - deletedProjectId: projectIdDeleted0, - }, - }, - { - _id: deleteProjectsRecordId1, - project: { - _id: projectIdDeleted1, - rootFolder: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [ - { _id: fileId5, hash: gitBlobHash(fileId5) }, - ], - folders: [], - }, - ], - }, - ], - }, - ], - overleaf: { history: { id: historyIdDeleted1 } }, - }, - deleterData: { - deletedProjectId: projectIdDeleted1, - }, - }, - { - _id: deleteProjectsRecordId2, - project: { - _id: projectIdNoHistoryDeleted, - rootFolder: [{ fileRefs: [], folders: [] }], - overleaf: { history: { conversionFailed: true } }, - }, - deleterData: { - deletedProjectId: projectIdNoHistoryDeleted, - }, - }, - { - _id: deleteProjectsRecordId3, - deleterData: { deletedProjectId: projectIdHardDeleted }, - }, - { - _id: deleteProjectsRecordId4, - project: { - _id: projectIdNoOverleafDeleted, - rootFolder: [{ fileRefs: [], folders: [] }], - }, - deleterData: { - deletedProjectId: projectIdNoOverleafDeleted, - }, - }, - ]) - }) - it('should have written the back filled files to history v1', async function () { - for (const { historyId, hash, fileId, content } of writtenBlobs) { - const blobStore = new BlobStore(historyId.toString()) - if (content) { - const s = await blobStore.getStream(hash) - const buf = new WritableBuffer() - await Stream.promises.pipeline(s, buf) - expect(buf.getContents()).to.deep.equal(content) - continue - } - const id = await blobStore.getString( - hash || gitBlobHash(fileId.toString()) - ) - expect(id).to.equal(fileId.toString()) - // double check we are not comparing 'undefined' or '[object Object]' above - expect(id).to.match(/^[a-f0-9]{24}$/) - } - }) - // Technically, we should move the below test into its own environment to ensure it does not impact any assertions. - // Practically, this is slow and moving it to the end of the tests gets us there most of the way. - it('should process nothing on re-run', async function () { - const rerun = await runScript( - !processHashedFiles ? ['--skip-hashed-files'] : [], - false - ) - expect(rerun.stats).deep.equal(statsForRerun(processHashedFiles)) - }) - } - - function expectNotFoundError(result, msg) { - expect(result.stdout).to.include(msg) - const log = JSON.parse( - result.stdout.split('\n').find(l => l.includes(`"${msg}"`)) - ) - expect(log).to.contain({ - projectId: projectId0.toString(), - fileId: fileId0.toString(), - path: 'rootFolder.0.fileRefs.1', - msg, - }) - expect(log.err).to.contain({ - name: 'NotFoundError', - }) - } - - const STATS_ALL_ZERO = { - projects: 0, - blobs: 0, - filesWithHash: 0, - filesWithoutHash: 0, - filesDuplicated: 0, - filesRetries: 0, - filesFailed: 0, - globalBlobsCount: 0, - globalBlobsEgress: 0, - fileTreeUpdated: 0, - projectDeleted: 0, - projectHardDeleted: 0, - fileHardDeleted: 0, - badFileTrees: 0, - mongoUpdates: 0, - readFromGCSCount: 0, - readFromGCSIngress: 0, - writeToGCSCount: 0, - writeToGCSEgress: 0, - } - const STATS_UP_TO_PROJECT1 = { - projects: 2, - blobs: 2, - filesWithHash: 0, - filesWithoutHash: 5, - filesDuplicated: 1, - filesRetries: 0, - filesFailed: 0, - globalBlobsCount: 1, - globalBlobsEgress: 30, - fileTreeUpdated: 0, - projectDeleted: 0, - projectHardDeleted: 0, - fileHardDeleted: 0, - badFileTrees: 0, - mongoUpdates: 2, // 4-2 blobs written to backedUpBlobs collection - readFromGCSCount: 4, - readFromGCSIngress: 4000072, - writeToGCSCount: 3, - writeToGCSEgress: 4000048, - } - const STATS_UP_FROM_PROJECT1_ONWARD = { - projects: 8, - blobs: 2, - filesWithHash: 0, - filesWithoutHash: 4, - filesDuplicated: 0, - filesRetries: 0, - filesFailed: 0, - globalBlobsCount: 0, - globalBlobsEgress: 0, - fileTreeUpdated: 0, - projectDeleted: 0, - projectHardDeleted: 0, - fileHardDeleted: 0, - badFileTrees: 4, - mongoUpdates: 3, // previously 5 blobs written to backedUpBlobs collection - readFromGCSCount: 4, - readFromGCSIngress: 96, - writeToGCSCount: 3, - writeToGCSEgress: 72, - } - const STATS_FILES_HASHED_EXTRA = { - ...STATS_ALL_ZERO, - filesWithHash: 2, - mongoUpdates: 0, // previously 2 blobs written to backedUpBlobs collection - readFromGCSCount: 2, - readFromGCSIngress: 48, - writeToGCSCount: 2, - writeToGCSEgress: 48, - } - - function sumStats(a, b) { - return Object.fromEntries(Object.entries(a).map(([k, v]) => [k, v + b[k]])) - } - - const STATS_ALL = sumStats( - STATS_UP_TO_PROJECT1, - STATS_UP_FROM_PROJECT1_ONWARD - ) - - function statsForRerun(processHashedFiles = true) { - let stats = { - ...STATS_ALL_ZERO, - // We still need to iterate over all the projects and blobs. - projects: 10, - blobs: 10, - - badFileTrees: 4, - } - if (processHashedFiles) { - stats = sumStats(stats, { - ...STATS_ALL_ZERO, - blobs: 2, - }) - } - return stats - } - - describe('error cases', function () { - beforeEach('prepare environment', prepareEnvironment) - - it('should gracefully handle fatal errors', async function () { - mockFilestore.deleteObject(projectId0, fileId0) - const t0 = Date.now() - const { stats, result } = await tryRunScript([ - '--skip-hashed-files', - '--retries=10', - '--retry-delay-ms=1000', - ]) - const t1 = Date.now() - expectNotFoundError(result, 'failed to process file') - expect(result.status).to.equal(1) - expect(stats).to.deep.equal( - sumStats(STATS_ALL, { - ...STATS_ALL_ZERO, - filesFailed: 1, - readFromGCSIngress: -24, - writeToGCSCount: -1, - writeToGCSEgress: -24, - }) - ) - // should not retry 404 - expect(result.stdout).to.not.include( - 'failed to process file, trying again' - ) - expect(t1 - t0).to.be.below(10_000) - expect(result.stderr).to.include( - 'The binary files migration failed, see above.' - ) - }) - - it('should retry on error', async function () { - mockFilestore.deleteObject(projectId0, fileId0) - const restoreFileAfter5s = async () => { - await setTimeout(5_000) - mockFilestore.addFile(projectId0, fileId0, fileId0.toString()) - } - // use Promise.allSettled to ensure the above sendStream call finishes before this test completes - const [ - { - value: { stats, result }, - }, - ] = await Promise.allSettled([ - tryRunScript([ - '--skip-hashed-files', - '--retries=60', - '--retry-delay-ms=1000', - '--retry-filestore-404', - ]), - restoreFileAfter5s(), - ]) - expectNotFoundError(result, 'failed to process file, trying again') - expect(result.status).to.equal(0) - expect({ ...stats, filesRetries: 0, readFromGCSCount: 0 }).to.deep.equal({ - ...STATS_ALL, - filesRetries: 0, - readFromGCSCount: 0, - }) - expect(stats.filesRetries).to.be.greaterThan(0, 'should have retried') - expect(stats.readFromGCSCount).to.be.greaterThan( - STATS_ALL.readFromGCSCount, - 'should have read more times from GCS compared to normal operations' - ) - }) - }) - - describe('full run CONCURRENCY=1', function () { - let output - before('prepare environment', prepareEnvironment) - before('run script', async function () { - output = await runScript(['--skip-hashed-files', '--concurrency=1']) - }) - - /** - * @param {ObjectId} projectId - * @param {string} msg - * @param {string} path - */ - function expectBadFileTreeMessage(projectId, msg, path) { - const line = output.result.stdout - .split('\n') - .find(l => l.includes(msg) && l.includes(projectId.toString())) - expect(line).to.exist - expect(JSON.parse(line)).to.include({ - projectId: projectId.toString(), - msg, - path, - }) - } - - it('should print stats', function () { - expect(output.stats).deep.equal(STATS_ALL) - }) - it('should print a warning message', () => { - expect(output.result.stderr).to.include( - 'The binary files migration succeeded on a subset of files' - ) - }) - it('should not create the migration', () => { - expect(output.migrationCreated).to.equal(false) - }) - it('should have logged the bad file-tree', function () { - expectBadFileTreeMessage( - projectIdBadFileTree0, - 'bad file-tree, bad folder', - 'rootFolder.0' - ) - expectBadFileTreeMessage( - projectIdBadFileTree1, - 'bad file-tree, bad folder', - 'rootFolder.0' - ) - expectBadFileTreeMessage( - projectIdBadFileTree1, - 'bad file-tree, bad folder', - 'rootFolder.0' - ) - expectBadFileTreeMessage( - projectIdBadFileTree2, - 'bad file-tree, bad fileRef id', - 'rootFolder.0.fileRefs.0' - ) - expectBadFileTreeMessage( - projectIdBadFileTree3, - 'bad file-tree, bad folder', - 'rootFolder.0.folders.0' - ) - expectBadFileTreeMessage( - projectIdBadFileTree3, - 'bad file-tree, bad folders', - 'rootFolder.0.folders.1.folders' - ) - expectBadFileTreeMessage( - projectIdBadFileTree3, - 'bad file-tree, bad fileRefs', - 'rootFolder.0.folders.1.fileRefs' - ) - }) - commonAssertions() - }) - - describe('when processing hashed files later', function () { - let output1, output2 - before('prepare environment', prepareEnvironment) - before('run script without hashed files', async function () { - output1 = await runScript(['--skip-hashed-files']) - }) - before('run script with hashed files', async function () { - output2 = await runScript([]) - }) - it('should print stats for the first run without hashed files', function () { - expect(output1.stats).deep.equal(STATS_ALL) - }) - it('should print stats for the hashed files run', function () { - expect(output2.stats).deep.equal({ - ...STATS_FILES_HASHED_EXTRA, - projects: 10, - blobs: 10, - badFileTrees: 4, - }) - }) - it('should print a success message', () => { - expect(output2.result.stderr).to.include( - 'The binary files migration succeeded.' - ) - }) - it('should create the migration', () => { - expect(output2.migrationCreated).to.equal(true) - }) - commonAssertions(true) - }) - describe('report mode', function () { - let output - before('prepare environment', prepareEnvironment) - before('run script', async function () { - output = await rawRunScript(['--report'], { LOG_LEVEL: 'error' }) - }) - it('should print the report', () => { - expect(output.status).to.equal(0) - expect(output.stdout).to.equal(`\ -Current status: -- Total number of projects: 10 -- Total number of deleted projects: 5 -Sampling 1000 projects to estimate progress... -Sampled stats for projects: -- Sampled projects: 9 (90% of all projects) -- Sampled projects with all hashes present: 5 -- Percentage of projects that need back-filling hashes: 44% (estimated) -- Sampled projects have 11 files that need to be checked against the full project history system. -- Sampled projects have 3 files that need to be uploaded to the full project history system (estimating 27% of all files). -Sampled stats for deleted projects: -- Sampled deleted projects: 4 (80% of all deleted projects) -- Sampled deleted projects with all hashes present: 3 -- Percentage of deleted projects that need back-filling hashes: 25% (estimated) -- Sampled deleted projects have 2 files that need to be checked against the full project history system. -- Sampled deleted projects have 1 files that need to be uploaded to the full project history system (estimating 50% of all files). -`) - }) - }) - - describe('full run in dry-run mode', function () { - let output - let projectRecordsBefore - let deletedProjectRecordsBefore - before('prepare environment', prepareEnvironment) - before(async function () { - projectRecordsBefore = await projectsCollection.find({}).toArray() - deletedProjectRecordsBefore = await deletedProjectsCollection - .find({}) - .toArray() - }) - before('run script', async function () { - output = await runScript(['--dry-run', '--concurrency=1'], false) - }) - - it('should print stats for dry-run mode', function () { - // Compute the stats for running the script without dry-run mode. - const originalStats = sumStats(STATS_ALL, { - ...STATS_FILES_HASHED_EXTRA, - readFromGCSCount: 30, - readFromGCSIngress: 72, - mongoUpdates: 0, - filesWithHash: 3, - }) - // For a dry-run mode, we expect the stats to be zero except for the - // count of projects, blobs, bad file trees, duplicated files - // and files with/without hash. All the other stats such as mongoUpdates - // and writeToGCSCount, etc should be zero. - const expectedDryRunStats = { - ...STATS_ALL_ZERO, - projects: originalStats.projects, - blobs: originalStats.blobs, - badFileTrees: originalStats.badFileTrees, - filesDuplicated: originalStats.filesDuplicated, - filesWithHash: originalStats.filesWithHash, - filesWithoutHash: originalStats.filesWithoutHash, - } - expect(output.stats).deep.equal(expectedDryRunStats) - }) - it('should not update mongo', async function () { - expect(await projectsCollection.find({}).toArray()).to.deep.equal( - projectRecordsBefore - ) - expect(await deletedProjectsCollection.find({}).toArray()).to.deep.equal( - deletedProjectRecordsBefore - ) - }) - }) - - describe('full run CONCURRENCY=10', function () { - let output - before('prepare environment', prepareEnvironment) - before('run script', async function () { - output = await runScript(['--skip-hashed-files', '--concurrency=10']) - }) - it('should print stats', function () { - expect(output.stats).deep.equal(STATS_ALL) - }) - commonAssertions() - }) - - describe('full run STREAM_HIGH_WATER_MARK=64kiB', function () { - let output - before('prepare environment', prepareEnvironment) - before('run script', async function () { - output = await runScript([ - '--skip-hashed-files', - `--stream-high-water-mark=${64 * 1024}`, - ]) - }) - it('should print stats', function () { - expect(output.stats).deep.equal(STATS_ALL) - }) - commonAssertions() - }) - - describe('when processing hashed files', function () { - let output - before('prepare environment', prepareEnvironment) - before('run script', async function () { - output = await runScript([]) - }) - it('should print stats', function () { - expect(output.stats).deep.equal( - sumStats(STATS_ALL, { - ...STATS_FILES_HASHED_EXTRA, - readFromGCSCount: 3, - readFromGCSIngress: 72, - mongoUpdates: 0, - filesWithHash: 3, - }) - ) - }) - commonAssertions(true) - }) - - describe('with something in the bucket already', function () { - before('prepare environment', prepareEnvironment) - before('create a file in s3', async function () { - const buf = Buffer.from(fileId0.toString()) - await backupPersistor.sendStream( - projectBlobsBucket, - makeProjectKey(historyId0, gitBlobHash(fileId0)), - Stream.Readable.from([buf]), - { contentLength: buf.byteLength } - ) - }) - let output - before('run script', async function () { - output = await runScript(['--skip-hashed-files', '--concurrency=1']) - }) - - it('should print stats', function () { - expect(output.stats).deep.equal( - sumStats(STATS_ALL, { - ...STATS_ALL_ZERO, - }) - ) - }) - commonAssertions() - }) - - describe('split run CONCURRENCY=1', function () { - // part0: project0+project1, part1: project2 onwards - const edge = projectId1.toString() - let outputPart0, outputPart1 - before('prepare environment', prepareEnvironment) - before('run script on part 0', async function () { - outputPart0 = await runScript([ - '--skip-hashed-files', - `--BATCH_RANGE_END=${edge}`, - '--concurrency=1', - ]) - }) - before('run script on part 1', async function () { - outputPart1 = await runScript([ - '--skip-hashed-files', - `--BATCH_RANGE_START=${edge}`, - '--concurrency=1', - ]) - }) - - it('should print stats for part 0', function () { - expect(outputPart0.stats).to.deep.equal(STATS_UP_TO_PROJECT1) - }) - it('should print stats for part 1', function () { - expect(outputPart1.stats).to.deep.equal(STATS_UP_FROM_PROJECT1_ONWARD) - }) - it('should warn about split run', () => { - expect(outputPart0.result.stderr).to.include( - 'The binary files migration succeeded on a subset of files' - ) - expect(outputPart1.result.stderr).to.include( - 'The binary files migration succeeded on a subset of files' - ) - }) - commonAssertions() - - describe('with a full run afterwards', () => { - let output - before('run script', async function () { - output = await runScript([]) - }) - it('should print stats', function () { - expect(output.stats).to.deep.equal( - sumStats(statsForRerun(false), STATS_FILES_HASHED_EXTRA) - ) - }) - it('should create the migration', () => { - expect(output.migrationCreated).to.equal(true) - }) - }) - }) - - describe('projectIds from file', function () { - const path0 = '/tmp/project-ids-0.txt' - const path1 = '/tmp/project-ids-1.txt' - before('prepare environment', prepareEnvironment) - before('create project-ids.txt files', async function () { - await fs.promises.writeFile( - path0, - [projectId0, projectId1].map(id => id.toString()).join('\n') - ) - await fs.promises.writeFile( - path1, - [ - projectId2, - projectId3, - projectIdDeleted0, - projectIdDeleted1, - projectIdNoHistory, - projectIdNoHistoryDeleted, - projectIdHardDeleted, - projectIdNoOverleaf, - projectIdNoOverleafDeleted, - projectIdBadFileTree0, - projectIdBadFileTree1, - projectIdBadFileTree2, - projectIdBadFileTree3, - ] - .map(id => id.toString()) - .join('\n') - ) - }) - - let outputPart0, outputPart1 - before('run script on part 0', async function () { - outputPart0 = await runScript([ - '--skip-hashed-files', - `--from-file=${path0}`, - ]) - }) - before('run script on part 1', async function () { - outputPart1 = await runScript([ - '--skip-hashed-files', - `--from-file=${path1}`, - ]) - }) - - /** - * @param {string} msg - * @param {ObjectId} projectId - */ - function expectLogEntry(msg, projectId) { - expect(outputPart1.result.stdout).to.include(msg) - const log = JSON.parse( - outputPart1.result.stdout - .split('\n') - .find(l => l.includes(`"${msg}"`) && l.includes(projectId.toString())) - ) - expect(log).to.contain({ - projectId: projectId.toString(), - msg, - }) - } - it('should flag the hard-deleted project', function () { - expectLogEntry('project hard-deleted', projectIdHardDeleted) - }) - it('should flag the projects without history id', function () { - expectLogEntry('project has no history id', projectIdNoOverleaf) - expectLogEntry('project has no history id', projectIdNoOverleafDeleted) - expectLogEntry('project has no history id', projectIdNoHistory) - expectLogEntry('project has no history id', projectIdNoHistoryDeleted) - }) - it('should print stats', function () { - expect(outputPart0.stats).to.deep.equal(STATS_UP_TO_PROJECT1) - expect(outputPart1.stats).to.deep.equal(STATS_UP_FROM_PROJECT1_ONWARD) - }) - it('should not create the migration ', () => { - expect(outputPart0.migrationCreated).to.equal(false) - expect(outputPart1.migrationCreated).to.equal(false) - }) - commonAssertions() - }) -}) diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs deleted file mode 100644 index 3aa00d685a..0000000000 --- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs +++ /dev/null @@ -1,621 +0,0 @@ -import fs from 'node:fs' -import Crypto from 'node:crypto' -import { promisify } from 'node:util' -import { Binary, ObjectId } from 'mongodb' -import { Blob } from 'overleaf-editor-core' -import { db } from '../../../../storage/lib/mongodb.js' -import cleanup from './support/cleanup.js' -import testProjects from '../api/support/test_projects.js' -import { execFile } from 'node:child_process' -import chai, { expect } from 'chai' -import chaiExclude from 'chai-exclude' -import { BlobStore } from '../../../../storage/lib/blob_store/index.js' -import { mockFilestore } from './support/MockFilestore.mjs' - -chai.use(chaiExclude) - -const TIMEOUT = 20 * 1_000 - -const projectsCollection = db.collection('projects') -const deletedProjectsCollection = db.collection('deletedProjects') - -/** - * @param {ObjectId} objectId - * @return {string} - */ -function gitBlobHash(objectId) { - return gitBlobHashBuffer(Buffer.from(objectId.toString())) -} - -/** - * @param {Buffer} buf - * @return {string} - */ -function gitBlobHashBuffer(buf) { - const sha = Crypto.createHash('sha1') - sha.update(`blob ${buf.byteLength}\x00`) - sha.update(buf) - return sha.digest('hex') -} - -/** - * @param {string} gitBlobHash - * @return {Binary} - */ -function binaryForGitBlobHash(gitBlobHash) { - return new Binary(Buffer.from(gitBlobHash, 'hex')) -} - -function objectIdFromTime(timestamp) { - return ObjectId.createFromTime(new Date(timestamp).getTime() / 1000) -} - -const PRINT_IDS_AND_HASHES_FOR_DEBUGGING = false - -describe('back_fill_file_hash_fix_up script', function () { - this.timeout(TIMEOUT) - const USER_FILES_BUCKET_NAME = 'fake-user-files-gcs' - - const projectId0 = objectIdFromTime('2017-01-01T00:00:00Z') - const projectIdDeleted0 = objectIdFromTime('2017-01-01T00:04:00Z') - const historyId0 = 42 // stored as number is mongo - const historyIdDeleted0 = projectIdDeleted0.toString() - const fileIdWithDifferentHashFound = objectIdFromTime('2017-02-01T00:00:00Z') - const fileIdInGoodState = objectIdFromTime('2017-02-01T00:01:00Z') - const fileIdWithDifferentHashNotFound0 = objectIdFromTime( - '2017-02-01T00:03:00Z' - ) - const fileIdWithDifferentHashNotFound1 = objectIdFromTime( - '2017-02-01T00:04:00Z' - ) - const fileIdBlobExistsInGCSCorrupted = objectIdFromTime( - '2017-02-01T00:05:00Z' - ) - const fileIdMissing0 = objectIdFromTime('2024-02-01T00:06:00Z') - const fileIdMissing1 = objectIdFromTime('2017-02-01T00:07:00Z') - const fileIdWithDifferentHashRestore = objectIdFromTime( - '2017-02-01T00:08:00Z' - ) - const fileIdMissing2 = objectIdFromTime('2017-02-01T00:12:00Z') - const fileIdHashMissing0 = objectIdFromTime('2017-02-01T00:13:00Z') - const fileIdHashMissing1 = objectIdFromTime('2017-02-01T00:14:00Z') - const contentCorruptedBlob = 'string that produces another hash' - const contentDoesNotExistAsBlob = 'does not exist as blob' - const hashDoesNotExistAsBlob = gitBlobHashBuffer( - Buffer.from(contentDoesNotExistAsBlob) - ) - const deleteProjectsRecordId0 = new ObjectId() - const writtenBlobs = [ - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdWithDifferentHashNotFound0, - }, - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdHashMissing0, - }, - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdHashMissing1, - }, - { - projectId: projectIdDeleted0, - historyId: historyIdDeleted0, - fileId: fileIdWithDifferentHashNotFound1, - }, - ] - const logs = [ - { - projectId: projectId0, - fileId: fileIdWithDifferentHashFound, - err: { message: 'OError: hash mismatch' }, - hash: gitBlobHash(fileIdMissing0), // does not matter - entry: { - ctx: { historyId: historyId0.toString() }, - hash: gitBlobHash(fileIdInGoodState), - }, - msg: 'failed to process file', - }, - { - projectId: projectId0, - fileId: fileIdWithDifferentHashRestore, - err: { message: 'OError: hash mismatch' }, - hash: hashDoesNotExistAsBlob, - entry: { - ctx: { historyId: historyId0.toString() }, - hash: gitBlobHash(fileIdMissing0), // does not matter - }, - msg: 'failed to process file', - }, - { - projectId: projectId0, - fileId: fileIdWithDifferentHashNotFound0, - err: { message: 'OError: hash mismatch' }, - hash: gitBlobHash(fileIdWithDifferentHashNotFound0), - entry: { - ctx: { historyId: historyId0.toString() }, - hash: hashDoesNotExistAsBlob, - }, - msg: 'failed to process file', - }, - { - projectId: projectIdDeleted0, - fileId: fileIdWithDifferentHashNotFound1, - err: { message: 'OError: hash mismatch' }, - hash: gitBlobHash(fileIdWithDifferentHashNotFound1), - entry: { - ctx: { historyId: historyIdDeleted0.toString() }, - hash: hashDoesNotExistAsBlob, - }, - msg: 'failed to process file', - }, - { - projectId: projectId0, - fileId: fileIdMissing0, - bucketName: USER_FILES_BUCKET_NAME, - err: { message: 'NotFoundError' }, - msg: 'failed to process file', - }, - { - projectId: projectId0, - fileId: fileIdMissing2, - bucketName: USER_FILES_BUCKET_NAME, - err: { message: 'NotFoundError' }, - msg: 'failed to process file', - }, - { - projectId: projectIdDeleted0, - fileId: fileIdMissing1, - bucketName: USER_FILES_BUCKET_NAME, - err: { message: 'NotFoundError' }, - msg: 'failed to process file', - }, - { - err: { message: 'spurious error' }, - msg: 'failed to process file, trying again', - }, - { - err: { message: 'some other error' }, - msg: 'failed to process file', - }, - // from find_malformed_filetrees.mjs - { - projectId: projectId0, - _id: fileIdHashMissing0, - reason: 'bad file hash', - msg: 'bad file-tree path', - }, - { - projectId: projectId0, - _id: fileIdHashMissing1, - reason: 'bad file hash', - msg: 'bad file-tree path', - }, - { - projectId: projectId0, - _id: fileIdBlobExistsInGCSCorrupted, - reason: 'bad file hash', - msg: 'bad file-tree path', - }, - ] - if (PRINT_IDS_AND_HASHES_FOR_DEBUGGING) { - const fileIds = { - fileIdWithDifferentHashFound, - fileIdInGoodState, - fileIdWithDifferentHashNotFound0, - fileIdWithDifferentHashNotFound1, - fileIdMissing0, - fileIdMissing1, - fileIdMissing2, - fileIdWithDifferentHashRestore, - fileIdHashMissing0, - fileIdHashMissing1, - } - console.log({ - projectId0, - projectIdDeleted0, - historyId0, - historyIdDeleted0, - ...fileIds, - hashDoesNotExistAsBlob, - }) - for (const [name, v] of Object.entries(fileIds)) { - console.log( - name, - gitBlobHash(v), - Array.from(binaryForGitBlobHash(gitBlobHash(v)).value()) - ) - } - } - - before(cleanup.everything) - - before('populate blobs/GCS', async function () { - await mockFilestore.start() - mockFilestore.addFile( - projectId0, - fileIdHashMissing0, - fileIdHashMissing0.toString() - ) - mockFilestore.addFile( - projectId0, - fileIdHashMissing1, - fileIdHashMissing1.toString() - ) - mockFilestore.addFile( - projectId0, - fileIdBlobExistsInGCSCorrupted, - fileIdBlobExistsInGCSCorrupted.toString() - ) - await new BlobStore(historyId0.toString()).putString( - fileIdHashMissing1.toString() // partially processed - ) - const path = '/tmp/test-blob-corrupted' - try { - await fs.promises.writeFile(path, contentCorruptedBlob) - await new BlobStore(historyId0.toString()).putBlob( - path, - new Blob(gitBlobHash(fileIdBlobExistsInGCSCorrupted), 42) - ) - } finally { - await fs.promises.rm(path, { force: true }) - } - await cleanup.postgres() - await cleanup.mongo() - await Promise.all([ - testProjects.createEmptyProject(historyId0.toString()), - testProjects.createEmptyProject(historyIdDeleted0), - ]) - await new BlobStore(historyId0.toString()).putString( - fileIdWithDifferentHashNotFound0.toString() - ) - await new BlobStore(historyIdDeleted0.toString()).putString( - fileIdWithDifferentHashNotFound1.toString() - ) - await new BlobStore(historyId0.toString()).putString( - fileIdInGoodState.toString() - ) - }) - - before('populate mongo', async function () { - await projectsCollection.insertMany([ - { - _id: projectId0, - rootFolder: [ - { - fileRefs: [ - { _id: fileIdMissing0 }, - { _id: fileIdMissing0 }, // bad file-tree, duplicated fileRef. - { _id: fileIdMissing2 }, - { _id: fileIdHashMissing0 }, - { _id: fileIdHashMissing1 }, - { - _id: fileIdWithDifferentHashFound, - hash: gitBlobHash(fileIdInGoodState), - }, - { - _id: fileIdWithDifferentHashRestore, - hash: gitBlobHash(fileIdMissing0), - }, - ], - folders: [ - { - docs: [], - }, - null, - { - fileRefs: [ - null, - { - _id: fileIdInGoodState, - hash: gitBlobHash(fileIdInGoodState), - }, - { - _id: fileIdWithDifferentHashNotFound0, - hash: hashDoesNotExistAsBlob, - }, - { - _id: fileIdBlobExistsInGCSCorrupted, - hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), - }, - ], - folders: [], - }, - ], - }, - ], - overleaf: { history: { id: historyId0 } }, - version: 0, - }, - ]) - await deletedProjectsCollection.insertMany([ - { - _id: deleteProjectsRecordId0, - project: { - _id: projectIdDeleted0, - rootFolder: [ - { - fileRefs: [ - { - _id: fileIdWithDifferentHashNotFound1, - hash: hashDoesNotExistAsBlob, - }, - ], - folders: [ - { - fileRefs: [], - folders: [ - { fileRefs: [{ _id: fileIdMissing1 }], folders: [] }, - ], - }, - ], - }, - ], - overleaf: { history: { id: historyIdDeleted0 } }, - version: 100, - }, - deleterData: { - deletedProjectId: projectIdDeleted0, - }, - }, - ]) - }) - - /** - * @param {Array} args - * @param {Record} env - * @return {Promise<{ stdout: string, stderr: string, status: number }>} - */ - async function tryRunScript(args = [], env = {}) { - let result - try { - result = await promisify(execFile)( - process.argv0, - ['storage/scripts/back_fill_file_hash_fix_up.mjs', ...args], - { - encoding: 'utf-8', - timeout: TIMEOUT - 500, - env: { - ...process.env, - USER_FILES_BUCKET_NAME, - SLEEP_BEFORE_EXIT: '1', - ...env, - LOG_LEVEL: 'warn', // Override LOG_LEVEL of acceptance tests - }, - } - ) - result.status = 0 - } catch (err) { - const { stdout, stderr, code } = err - if (typeof code !== 'number') { - console.log(err) - } - result = { stdout, stderr, status: code } - } - expect((await fs.promises.readdir('/tmp')).join(';')).to.not.match( - /back_fill_file_hash/ - ) - return result - } - async function runScriptWithLogs() { - const logsPath = '/tmp/test-script-logs' - let result - try { - await fs.promises.writeFile( - logsPath, - logs.map(e => JSON.stringify(e)).join('\n') - ) - result = await tryRunScript([`--logs=${logsPath}`]) - } finally { - await fs.promises.rm(logsPath, { force: true }) - } - const stats = JSON.parse(result.stdout.trim().split('\n').pop()) - return { - result, - stats, - } - } - - let result, stats - before(async function () { - ;({ result, stats } = await runScriptWithLogs()) - }) - it('should print stats', function () { - expect(stats).to.contain({ - processedLines: 12, - success: 7, - alreadyProcessed: 0, - fileDeleted: 0, - skipped: 0, - failed: 3, - unmatched: 1, - }) - }) - it('should handle re-run on same logs', async function () { - ;({ stats } = await runScriptWithLogs()) - expect(stats).to.contain({ - processedLines: 12, - success: 0, - alreadyProcessed: 4, - fileDeleted: 3, - skipped: 0, - failed: 3, - unmatched: 1, - }) - }) - it('should flag the unknown fatal error', function () { - const unknown = result.stdout - .split('\n') - .filter(l => l.includes('unknown fatal error')) - expect(unknown).to.have.length(1) - const [line] = unknown - expect(line).to.exist - expect(line).to.include('some other error') - }) - it('should flag the unexpected blob on mismatched hash', function () { - const line = result.stdout - .split('\n') - .find(l => l.includes('found blob with computed filestore object hash')) - expect(line).to.exist - expect(line).to.include(projectId0.toString()) - expect(line).to.include(fileIdWithDifferentHashFound.toString()) - expect(line).to.include(gitBlobHash(fileIdInGoodState)) - }) - it('should flag the need to restore', function () { - const line = result.stdout - .split('\n') - .find(l => l.includes('missing blob, need to restore filestore file')) - expect(line).to.exist - expect(line).to.include(projectId0.toString()) - expect(line).to.include(fileIdWithDifferentHashRestore.toString()) - expect(line).to.include(hashDoesNotExistAsBlob) - }) - it('should flag the corrupted blob', function () { - const line = result.stdout - .split('\n') - .find(l => l.includes('blob corrupted')) - expect(line).to.exist - expect(line).to.include(projectId0.toString()) - expect(line).to.include(fileIdBlobExistsInGCSCorrupted.toString()) - expect(line).to.include( - gitBlobHashBuffer(Buffer.from(contentCorruptedBlob)) - ) - expect(line).to.include(gitBlobHash(fileIdBlobExistsInGCSCorrupted)) - }) - it('should update mongo', async function () { - expect(await projectsCollection.find({}).toArray()) - .excludingEvery([ - 'currentEndTimestamp', - 'currentEndVersion', - 'updatedAt', - 'backup', - ]) - .to.deep.equal([ - { - _id: projectId0, - rootFolder: [ - { - fileRefs: [ - // Removed - // { _id: fileIdMissing0 }, - // Removed - // { _id: fileIdMissing2 }, - // Added hash - { - _id: fileIdHashMissing0, - hash: gitBlobHash(fileIdHashMissing0), - }, - // Added hash - { - _id: fileIdHashMissing1, - hash: gitBlobHash(fileIdHashMissing1), - }, - // No change, should warn about the find. - { - _id: fileIdWithDifferentHashFound, - hash: gitBlobHash(fileIdInGoodState), - }, - // No change, should warn about the need to restore. - { - _id: fileIdWithDifferentHashRestore, - hash: gitBlobHash(fileIdMissing0), - }, - ], - folders: [ - { - docs: [], - }, - null, - { - fileRefs: [ - null, - // No change - { - _id: fileIdInGoodState, - hash: gitBlobHash(fileIdInGoodState), - }, - // Updated hash - { - _id: fileIdWithDifferentHashNotFound0, - hash: gitBlobHash(fileIdWithDifferentHashNotFound0), - }, - // No change, flagged - { - _id: fileIdBlobExistsInGCSCorrupted, - hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), - }, - ], - folders: [], - }, - ], - }, - ], - overleaf: { history: { id: historyId0 } }, - // Incremented when removing file/updating hash - version: 5, - }, - ]) - expect(await deletedProjectsCollection.find({}).toArray()).to.deep.equal([ - { - _id: deleteProjectsRecordId0, - project: { - _id: projectIdDeleted0, - rootFolder: [ - { - fileRefs: [ - // Updated hash - { - _id: fileIdWithDifferentHashNotFound1, - hash: gitBlobHash(fileIdWithDifferentHashNotFound1), - }, - ], - folders: [ - { - fileRefs: [], - folders: [ - { - fileRefs: [ - // Removed - // { _id: fileIdMissing1 }, - ], - folders: [], - }, - ], - }, - ], - }, - ], - overleaf: { history: { id: historyIdDeleted0 } }, - // Incremented when removing file/updating hash - version: 102, - }, - deleterData: { - deletedProjectId: projectIdDeleted0, - }, - }, - ]) - const writtenBlobsByProject = new Map() - for (const { projectId, fileId } of writtenBlobs) { - writtenBlobsByProject.set( - projectId, - (writtenBlobsByProject.get(projectId) || []).concat([fileId]) - ) - } - }) - it('should have written the back filled files to history v1', async function () { - for (const { historyId, fileId } of writtenBlobs) { - const blobStore = new BlobStore(historyId.toString()) - const hash = gitBlobHash(fileId.toString()) - const blob = await blobStore.getBlob(hash) - expect(blob).to.exist - expect(blob.getByteLength()).to.equal(24) - const id = await blobStore.getString(hash) - expect(id).to.equal(fileId.toString()) - // double check we are not comparing 'undefined' or '[object Object]' above - expect(id).to.match(/^[a-f0-9]{24}$/) - } - }) -})