mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-12 13:13:45 +01:00
[e2e] cut down flakeyness (#35896)
* [saas-e2e] conversion: wait for initial compile before looking at PDF * [eslint-plugin] s/before/beforeWithReRunOnTestRetry/ in e2e tests * [server-ce] remove filestore migration scripts and E2E tests * [server-ce] test: trim down upgrading tests to 5.x -> latest GitOrigin-RevId: 2426ec4a4f3d67a5e0e7850ea84a942ae4619f94
This commit is contained in:
Jakob Ackermann
@@ -22,5 +22,6 @@ module.exports = {
|
|||||||
'sorted-keys-in-locales': require('./sorted-keys-in-locales'),
|
'sorted-keys-in-locales': require('./sorted-keys-in-locales'),
|
||||||
'locale-variables-match-en': require('./locale-variables-match-en'),
|
'locale-variables-match-en': require('./locale-variables-match-en'),
|
||||||
'no-orphan-locale-keys': require('./no-orphan-locale-keys'),
|
'no-orphan-locale-keys': require('./no-orphan-locale-keys'),
|
||||||
|
'no-mocha-before': require('./no-mocha-before'),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
const Path = require('node:path')
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Block bare mocha `before()` hooks in Cypress e2e suites.
|
||||||
|
*
|
||||||
|
* `before()` hooks run once per suite and do NOT re-run when a test retries,
|
||||||
|
* while `beforeEach`-based hooks (like `beforeWithReRunOnTestRetry`) do.
|
||||||
|
* Mixing the two also changes ordering: all `before()` hooks run ahead of
|
||||||
|
* every `beforeEach`, not in source order. This matters in the Server Pro
|
||||||
|
* e2e suite, where suites change the server config via `startWith()` while
|
||||||
|
* test setup runs in `beforeWithReRunOnTestRetry()`: when a later suite
|
||||||
|
* changes the config, both must re-run on retry and in registration order,
|
||||||
|
* otherwise a retried test runs against setup or config belonging to a
|
||||||
|
* different suite.
|
||||||
|
*
|
||||||
|
* The autofix rewrites `before(fn)` to `beforeWithReRunOnTestRetry(fn)` and
|
||||||
|
* adds the helper import when missing. Configure the helper location (repo
|
||||||
|
* root relative, no extension) via the `helperPath` option.
|
||||||
|
*/
|
||||||
|
module.exports = {
|
||||||
|
meta: {
|
||||||
|
type: 'suggestion',
|
||||||
|
docs: {
|
||||||
|
description:
|
||||||
|
'Disallow mocha before() hooks in favor of beforeWithReRunOnTestRetry()',
|
||||||
|
},
|
||||||
|
fixable: 'code',
|
||||||
|
schema: [
|
||||||
|
{
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
helperPath: { type: 'string' },
|
||||||
|
},
|
||||||
|
additionalProperties: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
messages: {
|
||||||
|
noMochaBefore:
|
||||||
|
'before() hooks do not re-run when a test retries, so retried tests run against stale setup. Use beforeWithReRunOnTestRetry() instead.',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
create(context) {
|
||||||
|
const helperPath = context.options[0]?.helperPath
|
||||||
|
let lastImport = null
|
||||||
|
let hasHelperImport = false
|
||||||
|
|
||||||
|
function helperImportSpecifier() {
|
||||||
|
const absolute = Path.resolve(context.cwd, helperPath)
|
||||||
|
let relative = Path.relative(
|
||||||
|
Path.dirname(Path.resolve(context.cwd, context.filename)),
|
||||||
|
absolute
|
||||||
|
)
|
||||||
|
relative = relative.split(Path.sep).join('/')
|
||||||
|
if (!relative.startsWith('.')) {
|
||||||
|
relative = `./${relative}`
|
||||||
|
}
|
||||||
|
return relative
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
ImportDeclaration(node) {
|
||||||
|
lastImport = node
|
||||||
|
if (
|
||||||
|
node.specifiers.some(
|
||||||
|
specifier =>
|
||||||
|
specifier.type === 'ImportSpecifier' &&
|
||||||
|
specifier.imported.name === 'beforeWithReRunOnTestRetry'
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
hasHelperImport = true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
CallExpression(node) {
|
||||||
|
if (
|
||||||
|
node.callee.type !== 'Identifier' ||
|
||||||
|
node.callee.name !== 'before'
|
||||||
|
) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Only the global mocha hook; skip locally defined/imported `before`
|
||||||
|
const variable = context.sourceCode
|
||||||
|
.getScope(node)
|
||||||
|
.references.find(ref => ref.identifier === node.callee)?.resolved
|
||||||
|
if (variable && variable.defs.length > 0) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const fixable = node.arguments.length === 1 && helperPath
|
||||||
|
context.report({
|
||||||
|
node: node.callee,
|
||||||
|
messageId: 'noMochaBefore',
|
||||||
|
fix: fixable
|
||||||
|
? fixer => {
|
||||||
|
const fixes = [
|
||||||
|
fixer.replaceText(node.callee, 'beforeWithReRunOnTestRetry'),
|
||||||
|
]
|
||||||
|
if (!hasHelperImport) {
|
||||||
|
const importText = `import { beforeWithReRunOnTestRetry } from '${helperImportSpecifier()}'`
|
||||||
|
fixes.push(
|
||||||
|
lastImport
|
||||||
|
? fixer.insertTextAfter(lastImport, `\n${importText}`)
|
||||||
|
: fixer.insertTextBeforeRange([0, 0], `${importText}\n`)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return fixes
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -14,6 +14,7 @@ import noConsecutiveSpacesInLocales from '../../no-consecutive-spaces-in-locales
|
|||||||
import noStraightApostrophesInLocales from '../../no-straight-apostrophes-in-locales.js'
|
import noStraightApostrophesInLocales from '../../no-straight-apostrophes-in-locales.js'
|
||||||
import frenchTypographyInLocales from '../../french-typography-in-locales.js'
|
import frenchTypographyInLocales from '../../french-typography-in-locales.js'
|
||||||
import sortedKeysInLocales from '../../sorted-keys-in-locales.js'
|
import sortedKeysInLocales from '../../sorted-keys-in-locales.js'
|
||||||
|
import noMochaBefore from '../../no-mocha-before.js'
|
||||||
|
|
||||||
const __filename = fileURLToPath(import.meta.url)
|
const __filename = fileURLToPath(import.meta.url)
|
||||||
|
|
||||||
@@ -546,3 +547,81 @@ jsonRuleTester.run('french-typography-in-locales', frenchTypographyInLocales, {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const noMochaBeforeOptions = [
|
||||||
|
{ helperPath: 'server-ce/test/helpers/beforeWithReRunOnTestRetry' },
|
||||||
|
]
|
||||||
|
|
||||||
|
ruleTester.run('no-mocha-before', noMochaBefore, {
|
||||||
|
valid: [
|
||||||
|
{
|
||||||
|
code: `beforeEach(function () { setup() })`,
|
||||||
|
options: noMochaBeforeOptions,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
code: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
|
||||||
|
beforeWithReRunOnTestRetry(function () { setup() })`,
|
||||||
|
options: noMochaBeforeOptions,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// locally defined `before` is not the mocha hook
|
||||||
|
code: `function before(fn) { fn() }
|
||||||
|
before(function () { setup() })`,
|
||||||
|
options: noMochaBeforeOptions,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
invalid: [
|
||||||
|
{
|
||||||
|
// rename + import added after the last existing import
|
||||||
|
filename: 'server-ce/test/foo.spec.ts',
|
||||||
|
code: `import { login } from './helpers/login'
|
||||||
|
before(async function () { this.timeout(1000) })`,
|
||||||
|
options: noMochaBeforeOptions,
|
||||||
|
errors: 1,
|
||||||
|
output: `import { login } from './helpers/login'
|
||||||
|
import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
|
||||||
|
beforeWithReRunOnTestRetry(async function () { this.timeout(1000) })`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// no imports yet: import inserted at the top
|
||||||
|
filename: 'server-ce/test/foo.spec.ts',
|
||||||
|
code: `before(function () { setup() })`,
|
||||||
|
options: noMochaBeforeOptions,
|
||||||
|
errors: 1,
|
||||||
|
output: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
|
||||||
|
beforeWithReRunOnTestRetry(function () { setup() })`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// helper already imported: rename only
|
||||||
|
filename: 'server-ce/test/foo.spec.ts',
|
||||||
|
code: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
|
||||||
|
before(function () { setup() })`,
|
||||||
|
options: noMochaBeforeOptions,
|
||||||
|
errors: 1,
|
||||||
|
output: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
|
||||||
|
beforeWithReRunOnTestRetry(function () { setup() })`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// file inside helpers/: relative path has no directory prefix
|
||||||
|
filename: 'server-ce/test/helpers/config.ts',
|
||||||
|
code: `before(function () { setup() })`,
|
||||||
|
options: noMochaBeforeOptions,
|
||||||
|
errors: 1,
|
||||||
|
output: `import { beforeWithReRunOnTestRetry } from './beforeWithReRunOnTestRetry'
|
||||||
|
beforeWithReRunOnTestRetry(function () { setup() })`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// two-argument mocha form: report without autofix
|
||||||
|
filename: 'server-ce/test/foo.spec.ts',
|
||||||
|
code: `before('named hook', function () { setup() })`,
|
||||||
|
options: noMochaBeforeOptions,
|
||||||
|
errors: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// no helperPath configured: report without autofix
|
||||||
|
filename: 'server-ce/test/foo.spec.ts',
|
||||||
|
code: `before(function () { setup() })`,
|
||||||
|
errors: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,629 +0,0 @@
|
|||||||
// @ts-check
|
|
||||||
import Events from 'node:events'
|
|
||||||
import fs from 'node:fs'
|
|
||||||
import Stream from 'node:stream'
|
|
||||||
import { ObjectId } from 'mongodb'
|
|
||||||
import logger from '@overleaf/logger'
|
|
||||||
import OError from '@overleaf/o-error'
|
|
||||||
import { Blob } from 'overleaf-editor-core'
|
|
||||||
import {
|
|
||||||
BlobStore,
|
|
||||||
getStringLengthOfFile,
|
|
||||||
makeBlobForFile,
|
|
||||||
} from '../lib/blob_store/index.js'
|
|
||||||
import { db } from '../lib/mongodb.js'
|
|
||||||
import commandLineArgs from 'command-line-args'
|
|
||||||
import readline from 'node:readline'
|
|
||||||
import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
|
|
||||||
import { setTimeout } from 'node:timers/promises'
|
|
||||||
|
|
||||||
// Silence warning.
|
|
||||||
Events.setMaxListeners(20)
|
|
||||||
|
|
||||||
// Enable caching for ObjectId.toString()
|
|
||||||
ObjectId.cacheHexString = true
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @typedef {import("mongodb").Collection} Collection
|
|
||||||
* @typedef {import("mongodb").Collection<Project>} ProjectsCollection
|
|
||||||
* @typedef {import("mongodb").Collection<{project: Project}>} DeletedProjectsCollection
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @typedef {Object} FileRef
|
|
||||||
* @property {ObjectId} _id
|
|
||||||
* @property {string} hash
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @typedef {Object} Folder
|
|
||||||
* @property {Array<Folder>} folders
|
|
||||||
* @property {Array<FileRef>} fileRefs
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @typedef {Object} Project
|
|
||||||
* @property {ObjectId} _id
|
|
||||||
* @property {Array<Folder>} rootFolder
|
|
||||||
* @property {{history: {id: (number|string)}}} overleaf
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_MISSING_HASH: boolean, LOGS: string}}
|
|
||||||
*/
|
|
||||||
function parseArgs() {
|
|
||||||
const args = commandLineArgs([
|
|
||||||
{ name: 'fixNotFound', type: String, defaultValue: 'true' },
|
|
||||||
{ name: 'fixHashMismatch', type: String, defaultValue: 'true' },
|
|
||||||
{ name: 'fixMissingHash', type: String, defaultValue: 'true' },
|
|
||||||
{ name: 'logs', type: String, defaultValue: '' },
|
|
||||||
])
|
|
||||||
/**
|
|
||||||
* commandLineArgs cannot handle --foo=false, so go the long way
|
|
||||||
* @param {string} name
|
|
||||||
* @return {boolean}
|
|
||||||
*/
|
|
||||||
function boolVal(name) {
|
|
||||||
const v = args[name]
|
|
||||||
if (['true', 'false'].includes(v)) return v === 'true'
|
|
||||||
throw new Error(`expected "true" or "false" for boolean option ${name}`)
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
FIX_HASH_MISMATCH: boolVal('fixNotFound'),
|
|
||||||
FIX_NOT_FOUND: boolVal('fixHashMismatch'),
|
|
||||||
FIX_MISSING_HASH: boolVal('fixMissingHash'),
|
|
||||||
LOGS: args.logs,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const { FIX_HASH_MISMATCH, FIX_NOT_FOUND, FIX_MISSING_HASH, LOGS } = parseArgs()
|
|
||||||
if (!LOGS) {
|
|
||||||
throw new Error('--logs parameter missing')
|
|
||||||
}
|
|
||||||
const BUFFER_DIR = fs.mkdtempSync(
|
|
||||||
process.env.BUFFER_DIR_PREFIX || '/tmp/back_fill_file_hash-'
|
|
||||||
)
|
|
||||||
const USER_FILES_BUCKET_NAME = process.env.USER_FILES_BUCKET_NAME || ''
|
|
||||||
if (!USER_FILES_BUCKET_NAME) {
|
|
||||||
throw new Error('env var USER_FILES_BUCKET_NAME is missing')
|
|
||||||
}
|
|
||||||
// https://nodejs.org/api/stream.html#streamgetdefaulthighwatermarkobjectmode
|
|
||||||
const STREAM_HIGH_WATER_MARK = parseInt(
|
|
||||||
process.env.STREAM_HIGH_WATER_MARK || (64 * 1024).toString(),
|
|
||||||
10
|
|
||||||
)
|
|
||||||
const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10)
|
|
||||||
|
|
||||||
// Filestore endpoint location
|
|
||||||
const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1'
|
|
||||||
const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009'
|
|
||||||
|
|
||||||
async function fetchFromFilestore(projectId, fileId) {
|
|
||||||
const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}`
|
|
||||||
const response = await fetch(url)
|
|
||||||
if (!response.ok) {
|
|
||||||
if (response.status === 404) {
|
|
||||||
throw new NotFoundError('file not found in filestore', {
|
|
||||||
status: response.status,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
const body = await response.text()
|
|
||||||
throw new OError('fetchFromFilestore failed', {
|
|
||||||
projectId,
|
|
||||||
fileId,
|
|
||||||
status: response.status,
|
|
||||||
body,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if (!response.body) {
|
|
||||||
throw new OError('fetchFromFilestore response has no body', {
|
|
||||||
projectId,
|
|
||||||
fileId,
|
|
||||||
status: response.status,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return response.body
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @type {ProjectsCollection} */
|
|
||||||
const projectsCollection = db.collection('projects')
|
|
||||||
/** @type {DeletedProjectsCollection} */
|
|
||||||
const deletedProjectsCollection = db.collection('deletedProjects')
|
|
||||||
|
|
||||||
let gracefulShutdownInitiated = false
|
|
||||||
|
|
||||||
process.on('SIGINT', handleSignal)
|
|
||||||
process.on('SIGTERM', handleSignal)
|
|
||||||
|
|
||||||
function handleSignal() {
|
|
||||||
gracefulShutdownInitiated = true
|
|
||||||
console.warn('graceful shutdown initiated, draining queue')
|
|
||||||
}
|
|
||||||
|
|
||||||
class FileDeletedError extends OError {}
|
|
||||||
|
|
||||||
/** @type {Map<string,{project: Project, projectSoftDeleted: boolean}>} */
|
|
||||||
const PROJECT_CACHE = new Map()
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} projectId
|
|
||||||
* @return {Promise<{project: Project, projectSoftDeleted: boolean}>}
|
|
||||||
*/
|
|
||||||
async function getProject(projectId) {
|
|
||||||
const cached = PROJECT_CACHE.get(projectId)
|
|
||||||
if (cached) return cached
|
|
||||||
|
|
||||||
let projectSoftDeleted
|
|
||||||
let project = await projectsCollection.findOne({
|
|
||||||
_id: new ObjectId(projectId),
|
|
||||||
})
|
|
||||||
if (project) {
|
|
||||||
projectSoftDeleted = false
|
|
||||||
} else {
|
|
||||||
const softDeleted = await deletedProjectsCollection.findOne({
|
|
||||||
'deleterData.deletedProjectId': new ObjectId(projectId),
|
|
||||||
project: { $exists: true },
|
|
||||||
})
|
|
||||||
if (!softDeleted) {
|
|
||||||
throw new OError('project hard-deleted')
|
|
||||||
}
|
|
||||||
project = softDeleted.project
|
|
||||||
projectSoftDeleted = true
|
|
||||||
}
|
|
||||||
PROJECT_CACHE.set(projectId, { projectSoftDeleted, project })
|
|
||||||
return { projectSoftDeleted, project }
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {Folder} folder
|
|
||||||
* @param {string} fileId
|
|
||||||
* @return {{path: string, fileRef: FileRef, folder: Folder}|null}
|
|
||||||
*/
|
|
||||||
function getFileTreePath(folder, fileId) {
|
|
||||||
if (!folder) return null
|
|
||||||
let idx = 0
|
|
||||||
if (Array.isArray(folder.fileRefs)) {
|
|
||||||
for (const fileRef of folder.fileRefs) {
|
|
||||||
if (fileRef?._id.toString() === fileId) {
|
|
||||||
return {
|
|
||||||
fileRef,
|
|
||||||
path: `.fileRefs.${idx}`,
|
|
||||||
folder,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
idx++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
idx = 0
|
|
||||||
if (Array.isArray(folder.folders)) {
|
|
||||||
for (const child of folder.folders) {
|
|
||||||
const match = getFileTreePath(child, fileId)
|
|
||||||
if (match) {
|
|
||||||
return {
|
|
||||||
fileRef: match.fileRef,
|
|
||||||
folder: match.folder,
|
|
||||||
path: `.folders.${idx}${match.path}`,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
idx++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} projectId
|
|
||||||
* @param {string} fileId
|
|
||||||
* @return {Promise<{fileRef: FileRef, folder: Folder, fullPath: string, query: Object, projectSoftDeleted: boolean}>}
|
|
||||||
*/
|
|
||||||
async function findFile(projectId, fileId) {
|
|
||||||
const { projectSoftDeleted, project } = await getProject(projectId)
|
|
||||||
const match = getFileTreePath(project.rootFolder[0], fileId)
|
|
||||||
if (!match) {
|
|
||||||
throw new FileDeletedError('file not found in file-tree', {
|
|
||||||
projectSoftDeleted,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
const { path, fileRef, folder } = match
|
|
||||||
let fullPath
|
|
||||||
let query
|
|
||||||
if (projectSoftDeleted) {
|
|
||||||
fullPath = `project.rootFolder.0${path}`
|
|
||||||
query = {
|
|
||||||
'deleterData.deletedProjectId': new ObjectId(projectId),
|
|
||||||
[`${fullPath}._id`]: new ObjectId(fileId),
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
fullPath = `rootFolder.0${path}`
|
|
||||||
query = {
|
|
||||||
_id: new ObjectId(projectId),
|
|
||||||
[`${fullPath}._id`]: new ObjectId(fileId),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
projectSoftDeleted,
|
|
||||||
query,
|
|
||||||
fullPath,
|
|
||||||
fileRef,
|
|
||||||
folder,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} line
|
|
||||||
* @return {Promise<boolean>}
|
|
||||||
*/
|
|
||||||
async function fixNotFound(line) {
|
|
||||||
const { projectId, fileId, bucketName } = JSON.parse(line)
|
|
||||||
if (bucketName !== USER_FILES_BUCKET_NAME) {
|
|
||||||
throw new OError('not found case for another bucket')
|
|
||||||
}
|
|
||||||
|
|
||||||
const { projectSoftDeleted, query, fullPath, fileRef, folder } =
|
|
||||||
await findFile(projectId, fileId)
|
|
||||||
logger.info({ projectId, fileId, fileRef }, 'removing fileRef')
|
|
||||||
// Copied from _removeElementFromMongoArray (https://github.com/overleaf/internal/blob/11e09528c153de6b7766d18c3c90d94962190371/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js)
|
|
||||||
const nonArrayPath = fullPath.slice(0, fullPath.lastIndexOf('.'))
|
|
||||||
let result
|
|
||||||
if (projectSoftDeleted) {
|
|
||||||
result = await deletedProjectsCollection.updateOne(query, {
|
|
||||||
$pull: { [nonArrayPath]: { _id: new ObjectId(fileId) } },
|
|
||||||
$inc: { 'project.version': 1 },
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
result = await projectsCollection.updateOne(query, {
|
|
||||||
$pull: { [nonArrayPath]: { _id: new ObjectId(fileId) } },
|
|
||||||
$inc: { version: 1 },
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if (result.matchedCount !== 1) {
|
|
||||||
throw new OError('file-tree write did not match', { result })
|
|
||||||
}
|
|
||||||
// Update the cache. The mongo-path of the next file will be off otherwise.
|
|
||||||
folder.fileRefs = folder.fileRefs.filter(f => !f._id.equals(fileId))
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} projectId
|
|
||||||
* @param {string} fileId
|
|
||||||
* @param {string} hash
|
|
||||||
* @return {Promise<void>}
|
|
||||||
*/
|
|
||||||
async function setHashInMongo(projectId, fileId, hash) {
|
|
||||||
const { projectSoftDeleted, query, fullPath, fileRef } = await findFile(
|
|
||||||
projectId,
|
|
||||||
fileId
|
|
||||||
)
|
|
||||||
if (fileRef.hash === hash) return
|
|
||||||
logger.info({ projectId, fileId, fileRef, hash }, 'setting fileRef hash')
|
|
||||||
let result
|
|
||||||
if (projectSoftDeleted) {
|
|
||||||
result = await deletedProjectsCollection.updateOne(query, {
|
|
||||||
$set: { [`${fullPath}.hash`]: hash },
|
|
||||||
$inc: { 'project.version': 1 },
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
result = await projectsCollection.updateOne(query, {
|
|
||||||
$set: { [`${fullPath}.hash`]: hash },
|
|
||||||
$inc: { version: 1 },
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if (result.matchedCount !== 1) {
|
|
||||||
throw new OError('file-tree write did not match', { result })
|
|
||||||
}
|
|
||||||
fileRef.hash = hash // Update cache for completeness.
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} projectId
|
|
||||||
* @param {string} fileId
|
|
||||||
* @param {string} historyId
|
|
||||||
* @return {Promise<void>}
|
|
||||||
*/
|
|
||||||
async function importRestoredFilestoreFile(projectId, fileId, historyId) {
|
|
||||||
const path = `${BUFFER_DIR}/${projectId}_${fileId}`
|
|
||||||
try {
|
|
||||||
let s
|
|
||||||
try {
|
|
||||||
s = await fetchFromFilestore(projectId, fileId)
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof NotFoundError) {
|
|
||||||
throw new OError('missing blob, need to restore filestore file', {
|
|
||||||
projectId,
|
|
||||||
fileId,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
await Stream.promises.pipeline(
|
|
||||||
s,
|
|
||||||
fs.createWriteStream(path, { highWaterMark: STREAM_HIGH_WATER_MARK })
|
|
||||||
)
|
|
||||||
const blobStore = new BlobStore(historyId)
|
|
||||||
const blob = await blobStore.putFile(path)
|
|
||||||
await setHashInMongo(projectId, fileId, blob.getHash())
|
|
||||||
} finally {
|
|
||||||
await fs.promises.rm(path, { force: true })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} projectId
|
|
||||||
* @param {string} fileId
|
|
||||||
* @param {string} path
|
|
||||||
* @return {Promise<Blob>}
|
|
||||||
*/
|
|
||||||
async function bufferFilestoreFileToDisk(projectId, fileId, path) {
|
|
||||||
try {
|
|
||||||
await Stream.promises.pipeline(
|
|
||||||
await fetchFromFilestore(projectId, fileId),
|
|
||||||
fs.createWriteStream(path, { highWaterMark: STREAM_HIGH_WATER_MARK })
|
|
||||||
)
|
|
||||||
const blob = await makeBlobForFile(path)
|
|
||||||
blob.setStringLength(
|
|
||||||
await getStringLengthOfFile(blob.getByteLength(), path)
|
|
||||||
)
|
|
||||||
return blob
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof NotFoundError) {
|
|
||||||
throw new OError('missing blob, need to restore filestore file', {
|
|
||||||
projectId,
|
|
||||||
fileId,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} projectId
|
|
||||||
* @param {string} fileId
|
|
||||||
* @return {Promise<string>}
|
|
||||||
*/
|
|
||||||
async function computeFilestoreFileHash(projectId, fileId) {
|
|
||||||
const path = `${BUFFER_DIR}/${projectId}_${fileId}`
|
|
||||||
try {
|
|
||||||
const blob = await bufferFilestoreFileToDisk(projectId, fileId, path)
|
|
||||||
return blob.getHash()
|
|
||||||
} finally {
|
|
||||||
await fs.promises.rm(path, { force: true })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} projectId
|
|
||||||
* @param {string} fileId
|
|
||||||
* @return {Promise<void>}
|
|
||||||
*/
|
|
||||||
async function uploadFilestoreFile(projectId, fileId) {
|
|
||||||
const path = `${BUFFER_DIR}/${projectId}_${fileId}`
|
|
||||||
try {
|
|
||||||
const blob = await bufferFilestoreFileToDisk(projectId, fileId, path)
|
|
||||||
const hash = blob.getHash()
|
|
||||||
try {
|
|
||||||
await ensureBlobExistsForFile(projectId, fileId, hash)
|
|
||||||
} catch (err) {
|
|
||||||
if (!(err instanceof Blob.NotFoundError)) throw err
|
|
||||||
|
|
||||||
const { project } = await getProject(projectId)
|
|
||||||
const historyId = project.overleaf.history.id.toString()
|
|
||||||
const blobStore = new BlobStore(historyId)
|
|
||||||
await blobStore.putBlob(path, blob)
|
|
||||||
await ensureBlobExistsForFile(projectId, fileId, hash)
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
await fs.promises.rm(path, { force: true })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} line
|
|
||||||
* @return {Promise<boolean>}
|
|
||||||
*/
|
|
||||||
async function fixHashMismatch(line) {
|
|
||||||
const {
|
|
||||||
projectId,
|
|
||||||
fileId,
|
|
||||||
hash: computedHash,
|
|
||||||
entry: {
|
|
||||||
hash: fileTreeHash,
|
|
||||||
ctx: { historyId },
|
|
||||||
},
|
|
||||||
} = JSON.parse(line)
|
|
||||||
const blobStore = new BlobStore(historyId)
|
|
||||||
if (await blobStore.getBlob(fileTreeHash)) {
|
|
||||||
throw new OError('found blob with computed filestore object hash')
|
|
||||||
}
|
|
||||||
if (!(await blobStore.getBlob(computedHash))) {
|
|
||||||
await importRestoredFilestoreFile(projectId, fileId, historyId)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return await ensureBlobExistsForFile(projectId, fileId, computedHash)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} projectId
|
|
||||||
* @param {string} fileId
|
|
||||||
* @param {string} hash
|
|
||||||
* @return {Promise<boolean>}
|
|
||||||
*/
|
|
||||||
async function hashAlreadyUpdatedInFileTree(projectId, fileId, hash) {
|
|
||||||
const { fileRef } = await findFile(projectId, fileId)
|
|
||||||
return fileRef.hash === hash
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} projectId
|
|
||||||
* @param {string} fileId
|
|
||||||
* @param {string} hash
|
|
||||||
* @return {Promise<boolean>}
|
|
||||||
*/
|
|
||||||
async function ensureBlobExistsForFile(projectId, fileId, hash) {
|
|
||||||
const { project } = await getProject(projectId)
|
|
||||||
const historyId = project.overleaf.history.id.toString()
|
|
||||||
const blobStore = new BlobStore(historyId)
|
|
||||||
if (
|
|
||||||
(await hashAlreadyUpdatedInFileTree(projectId, fileId, hash)) &&
|
|
||||||
(await blobStore.getBlob(hash))
|
|
||||||
) {
|
|
||||||
return false // already processed
|
|
||||||
}
|
|
||||||
|
|
||||||
const stream = await blobStore.getStream(hash)
|
|
||||||
const path = `${BUFFER_DIR}/${historyId}_${hash}`
|
|
||||||
try {
|
|
||||||
await Stream.promises.pipeline(
|
|
||||||
stream,
|
|
||||||
fs.createWriteStream(path, {
|
|
||||||
highWaterMark: STREAM_HIGH_WATER_MARK,
|
|
||||||
})
|
|
||||||
)
|
|
||||||
|
|
||||||
const writtenBlob = await makeBlobForFile(path)
|
|
||||||
writtenBlob.setStringLength(
|
|
||||||
await getStringLengthOfFile(writtenBlob.getByteLength(), path)
|
|
||||||
)
|
|
||||||
if (writtenBlob.getHash() !== hash) {
|
|
||||||
// Double check download, better safe than sorry.
|
|
||||||
throw new OError('blob corrupted', { writtenBlob, hash })
|
|
||||||
}
|
|
||||||
|
|
||||||
let blob = await blobStore.getBlob(hash)
|
|
||||||
if (!blob) {
|
|
||||||
// Calling blobStore.putBlob would result in the same error again.
|
|
||||||
// HACK: Skip upload to GCS and finalize putBlob operation directly.
|
|
||||||
await blobStore.backend.insertBlob(historyId, writtenBlob)
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
await fs.promises.rm(path, { force: true })
|
|
||||||
}
|
|
||||||
await setHashInMongo(projectId, fileId, hash)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} line
|
|
||||||
* @return {Promise<boolean>}
|
|
||||||
*/
|
|
||||||
async function fixMissingHash(line) {
|
|
||||||
let { projectId, _id: fileId } = JSON.parse(line)
|
|
||||||
const {
|
|
||||||
fileRef: { hash },
|
|
||||||
} = await findFile(projectId, fileId)
|
|
||||||
if (hash) {
|
|
||||||
// processed, double check
|
|
||||||
return await ensureBlobExistsForFile(projectId, fileId, hash)
|
|
||||||
}
|
|
||||||
await uploadFilestoreFile(projectId, fileId)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
const CASES = {
|
|
||||||
'not found': {
|
|
||||||
match: 'NotFoundError',
|
|
||||||
flag: FIX_NOT_FOUND,
|
|
||||||
action: fixNotFound,
|
|
||||||
},
|
|
||||||
'hash mismatch': {
|
|
||||||
match: 'OError: hash mismatch',
|
|
||||||
flag: FIX_HASH_MISMATCH,
|
|
||||||
action: fixHashMismatch,
|
|
||||||
},
|
|
||||||
'missing file hash': {
|
|
||||||
match: '"bad file hash"',
|
|
||||||
flag: FIX_MISSING_HASH,
|
|
||||||
action: fixMissingHash,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
const STATS = {
|
|
||||||
processedLines: 0,
|
|
||||||
success: 0,
|
|
||||||
alreadyProcessed: 0,
|
|
||||||
fileDeleted: 0,
|
|
||||||
skipped: 0,
|
|
||||||
failed: 0,
|
|
||||||
unmatched: 0,
|
|
||||||
}
|
|
||||||
function logStats() {
|
|
||||||
console.log(
|
|
||||||
JSON.stringify({
|
|
||||||
time: new Date(),
|
|
||||||
gracefulShutdownInitiated,
|
|
||||||
...STATS,
|
|
||||||
})
|
|
||||||
)
|
|
||||||
}
|
|
||||||
setInterval(logStats, 10_000)
|
|
||||||
|
|
||||||
async function processLog() {
|
|
||||||
const rl = readline.createInterface({
|
|
||||||
input: fs.createReadStream(LOGS),
|
|
||||||
})
|
|
||||||
nextLine: for await (const line of rl) {
|
|
||||||
if (gracefulShutdownInitiated) break
|
|
||||||
STATS.processedLines++
|
|
||||||
if (
|
|
||||||
!(
|
|
||||||
line.includes('"failed to process file"') ||
|
|
||||||
// Process missing hashes as flagged by find_malformed_filetrees.mjs
|
|
||||||
line.includes('"bad file-tree path"')
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const [name, { match, flag, action }] of Object.entries(CASES)) {
|
|
||||||
if (!line.includes(match)) continue
|
|
||||||
if (flag) {
|
|
||||||
try {
|
|
||||||
if (await action(line)) {
|
|
||||||
STATS.success++
|
|
||||||
} else {
|
|
||||||
STATS.alreadyProcessed++
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof FileDeletedError) {
|
|
||||||
STATS.fileDeleted++
|
|
||||||
logger.info({ err, line }, 'file deleted, skipping')
|
|
||||||
} else {
|
|
||||||
STATS.failed++
|
|
||||||
logger.error({ err, line }, `failed to fix ${name}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
STATS.skipped++
|
|
||||||
}
|
|
||||||
continue nextLine
|
|
||||||
}
|
|
||||||
STATS.unmatched++
|
|
||||||
logger.warn({ line }, 'unknown fatal error')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
try {
|
|
||||||
await processLog()
|
|
||||||
} finally {
|
|
||||||
logStats()
|
|
||||||
try {
|
|
||||||
await fs.promises.rm(BUFFER_DIR, { recursive: true, force: true })
|
|
||||||
} catch (err) {
|
|
||||||
console.error(`Cleanup of BUFFER_DIR=${BUFFER_DIR} failed`, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const { skipped, failed, unmatched } = STATS
|
|
||||||
await setTimeout(SLEEP_BEFORE_EXIT)
|
|
||||||
if (failed > 0) {
|
|
||||||
process.exit(Math.min(failed, 99))
|
|
||||||
} else if (unmatched > 0) {
|
|
||||||
process.exit(100)
|
|
||||||
} else if (skipped > 0) {
|
|
||||||
process.exit(101)
|
|
||||||
} else {
|
|
||||||
process.exit(0)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
await main()
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,621 +0,0 @@
|
|||||||
import fs from 'node:fs'
|
|
||||||
import Crypto from 'node:crypto'
|
|
||||||
import { promisify } from 'node:util'
|
|
||||||
import { Binary, ObjectId } from 'mongodb'
|
|
||||||
import { Blob } from 'overleaf-editor-core'
|
|
||||||
import { db } from '../../../../storage/lib/mongodb.js'
|
|
||||||
import cleanup from './support/cleanup.js'
|
|
||||||
import testProjects from '../api/support/test_projects.js'
|
|
||||||
import { execFile } from 'node:child_process'
|
|
||||||
import chai, { expect } from 'chai'
|
|
||||||
import chaiExclude from 'chai-exclude'
|
|
||||||
import { BlobStore } from '../../../../storage/lib/blob_store/index.js'
|
|
||||||
import { mockFilestore } from './support/MockFilestore.mjs'
|
|
||||||
|
|
||||||
chai.use(chaiExclude)
|
|
||||||
|
|
||||||
const TIMEOUT = 20 * 1_000
|
|
||||||
|
|
||||||
const projectsCollection = db.collection('projects')
|
|
||||||
const deletedProjectsCollection = db.collection('deletedProjects')
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {ObjectId} objectId
|
|
||||||
* @return {string}
|
|
||||||
*/
|
|
||||||
function gitBlobHash(objectId) {
|
|
||||||
return gitBlobHashBuffer(Buffer.from(objectId.toString()))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {Buffer} buf
|
|
||||||
* @return {string}
|
|
||||||
*/
|
|
||||||
function gitBlobHashBuffer(buf) {
|
|
||||||
const sha = Crypto.createHash('sha1')
|
|
||||||
sha.update(`blob ${buf.byteLength}\x00`)
|
|
||||||
sha.update(buf)
|
|
||||||
return sha.digest('hex')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} gitBlobHash
|
|
||||||
* @return {Binary}
|
|
||||||
*/
|
|
||||||
function binaryForGitBlobHash(gitBlobHash) {
|
|
||||||
return new Binary(Buffer.from(gitBlobHash, 'hex'))
|
|
||||||
}
|
|
||||||
|
|
||||||
function objectIdFromTime(timestamp) {
|
|
||||||
return ObjectId.createFromTime(new Date(timestamp).getTime() / 1000)
|
|
||||||
}
|
|
||||||
|
|
||||||
const PRINT_IDS_AND_HASHES_FOR_DEBUGGING = false
|
|
||||||
|
|
||||||
describe('back_fill_file_hash_fix_up script', function () {
|
|
||||||
this.timeout(TIMEOUT)
|
|
||||||
const USER_FILES_BUCKET_NAME = 'fake-user-files-gcs'
|
|
||||||
|
|
||||||
const projectId0 = objectIdFromTime('2017-01-01T00:00:00Z')
|
|
||||||
const projectIdDeleted0 = objectIdFromTime('2017-01-01T00:04:00Z')
|
|
||||||
const historyId0 = 42 // stored as number is mongo
|
|
||||||
const historyIdDeleted0 = projectIdDeleted0.toString()
|
|
||||||
const fileIdWithDifferentHashFound = objectIdFromTime('2017-02-01T00:00:00Z')
|
|
||||||
const fileIdInGoodState = objectIdFromTime('2017-02-01T00:01:00Z')
|
|
||||||
const fileIdWithDifferentHashNotFound0 = objectIdFromTime(
|
|
||||||
'2017-02-01T00:03:00Z'
|
|
||||||
)
|
|
||||||
const fileIdWithDifferentHashNotFound1 = objectIdFromTime(
|
|
||||||
'2017-02-01T00:04:00Z'
|
|
||||||
)
|
|
||||||
const fileIdBlobExistsInGCSCorrupted = objectIdFromTime(
|
|
||||||
'2017-02-01T00:05:00Z'
|
|
||||||
)
|
|
||||||
const fileIdMissing0 = objectIdFromTime('2024-02-01T00:06:00Z')
|
|
||||||
const fileIdMissing1 = objectIdFromTime('2017-02-01T00:07:00Z')
|
|
||||||
const fileIdWithDifferentHashRestore = objectIdFromTime(
|
|
||||||
'2017-02-01T00:08:00Z'
|
|
||||||
)
|
|
||||||
const fileIdMissing2 = objectIdFromTime('2017-02-01T00:12:00Z')
|
|
||||||
const fileIdHashMissing0 = objectIdFromTime('2017-02-01T00:13:00Z')
|
|
||||||
const fileIdHashMissing1 = objectIdFromTime('2017-02-01T00:14:00Z')
|
|
||||||
const contentCorruptedBlob = 'string that produces another hash'
|
|
||||||
const contentDoesNotExistAsBlob = 'does not exist as blob'
|
|
||||||
const hashDoesNotExistAsBlob = gitBlobHashBuffer(
|
|
||||||
Buffer.from(contentDoesNotExistAsBlob)
|
|
||||||
)
|
|
||||||
const deleteProjectsRecordId0 = new ObjectId()
|
|
||||||
const writtenBlobs = [
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
historyId: historyId0,
|
|
||||||
fileId: fileIdWithDifferentHashNotFound0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
historyId: historyId0,
|
|
||||||
fileId: fileIdHashMissing0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
historyId: historyId0,
|
|
||||||
fileId: fileIdHashMissing1,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectIdDeleted0,
|
|
||||||
historyId: historyIdDeleted0,
|
|
||||||
fileId: fileIdWithDifferentHashNotFound1,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
const logs = [
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
fileId: fileIdWithDifferentHashFound,
|
|
||||||
err: { message: 'OError: hash mismatch' },
|
|
||||||
hash: gitBlobHash(fileIdMissing0), // does not matter
|
|
||||||
entry: {
|
|
||||||
ctx: { historyId: historyId0.toString() },
|
|
||||||
hash: gitBlobHash(fileIdInGoodState),
|
|
||||||
},
|
|
||||||
msg: 'failed to process file',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
fileId: fileIdWithDifferentHashRestore,
|
|
||||||
err: { message: 'OError: hash mismatch' },
|
|
||||||
hash: hashDoesNotExistAsBlob,
|
|
||||||
entry: {
|
|
||||||
ctx: { historyId: historyId0.toString() },
|
|
||||||
hash: gitBlobHash(fileIdMissing0), // does not matter
|
|
||||||
},
|
|
||||||
msg: 'failed to process file',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
fileId: fileIdWithDifferentHashNotFound0,
|
|
||||||
err: { message: 'OError: hash mismatch' },
|
|
||||||
hash: gitBlobHash(fileIdWithDifferentHashNotFound0),
|
|
||||||
entry: {
|
|
||||||
ctx: { historyId: historyId0.toString() },
|
|
||||||
hash: hashDoesNotExistAsBlob,
|
|
||||||
},
|
|
||||||
msg: 'failed to process file',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectIdDeleted0,
|
|
||||||
fileId: fileIdWithDifferentHashNotFound1,
|
|
||||||
err: { message: 'OError: hash mismatch' },
|
|
||||||
hash: gitBlobHash(fileIdWithDifferentHashNotFound1),
|
|
||||||
entry: {
|
|
||||||
ctx: { historyId: historyIdDeleted0.toString() },
|
|
||||||
hash: hashDoesNotExistAsBlob,
|
|
||||||
},
|
|
||||||
msg: 'failed to process file',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
fileId: fileIdMissing0,
|
|
||||||
bucketName: USER_FILES_BUCKET_NAME,
|
|
||||||
err: { message: 'NotFoundError' },
|
|
||||||
msg: 'failed to process file',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
fileId: fileIdMissing2,
|
|
||||||
bucketName: USER_FILES_BUCKET_NAME,
|
|
||||||
err: { message: 'NotFoundError' },
|
|
||||||
msg: 'failed to process file',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectIdDeleted0,
|
|
||||||
fileId: fileIdMissing1,
|
|
||||||
bucketName: USER_FILES_BUCKET_NAME,
|
|
||||||
err: { message: 'NotFoundError' },
|
|
||||||
msg: 'failed to process file',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
err: { message: 'spurious error' },
|
|
||||||
msg: 'failed to process file, trying again',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
err: { message: 'some other error' },
|
|
||||||
msg: 'failed to process file',
|
|
||||||
},
|
|
||||||
// from find_malformed_filetrees.mjs
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
_id: fileIdHashMissing0,
|
|
||||||
reason: 'bad file hash',
|
|
||||||
msg: 'bad file-tree path',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
_id: fileIdHashMissing1,
|
|
||||||
reason: 'bad file hash',
|
|
||||||
msg: 'bad file-tree path',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
projectId: projectId0,
|
|
||||||
_id: fileIdBlobExistsInGCSCorrupted,
|
|
||||||
reason: 'bad file hash',
|
|
||||||
msg: 'bad file-tree path',
|
|
||||||
},
|
|
||||||
]
|
|
||||||
if (PRINT_IDS_AND_HASHES_FOR_DEBUGGING) {
|
|
||||||
const fileIds = {
|
|
||||||
fileIdWithDifferentHashFound,
|
|
||||||
fileIdInGoodState,
|
|
||||||
fileIdWithDifferentHashNotFound0,
|
|
||||||
fileIdWithDifferentHashNotFound1,
|
|
||||||
fileIdMissing0,
|
|
||||||
fileIdMissing1,
|
|
||||||
fileIdMissing2,
|
|
||||||
fileIdWithDifferentHashRestore,
|
|
||||||
fileIdHashMissing0,
|
|
||||||
fileIdHashMissing1,
|
|
||||||
}
|
|
||||||
console.log({
|
|
||||||
projectId0,
|
|
||||||
projectIdDeleted0,
|
|
||||||
historyId0,
|
|
||||||
historyIdDeleted0,
|
|
||||||
...fileIds,
|
|
||||||
hashDoesNotExistAsBlob,
|
|
||||||
})
|
|
||||||
for (const [name, v] of Object.entries(fileIds)) {
|
|
||||||
console.log(
|
|
||||||
name,
|
|
||||||
gitBlobHash(v),
|
|
||||||
Array.from(binaryForGitBlobHash(gitBlobHash(v)).value())
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
before(cleanup.everything)
|
|
||||||
|
|
||||||
before('populate blobs/GCS', async function () {
|
|
||||||
await mockFilestore.start()
|
|
||||||
mockFilestore.addFile(
|
|
||||||
projectId0,
|
|
||||||
fileIdHashMissing0,
|
|
||||||
fileIdHashMissing0.toString()
|
|
||||||
)
|
|
||||||
mockFilestore.addFile(
|
|
||||||
projectId0,
|
|
||||||
fileIdHashMissing1,
|
|
||||||
fileIdHashMissing1.toString()
|
|
||||||
)
|
|
||||||
mockFilestore.addFile(
|
|
||||||
projectId0,
|
|
||||||
fileIdBlobExistsInGCSCorrupted,
|
|
||||||
fileIdBlobExistsInGCSCorrupted.toString()
|
|
||||||
)
|
|
||||||
await new BlobStore(historyId0.toString()).putString(
|
|
||||||
fileIdHashMissing1.toString() // partially processed
|
|
||||||
)
|
|
||||||
const path = '/tmp/test-blob-corrupted'
|
|
||||||
try {
|
|
||||||
await fs.promises.writeFile(path, contentCorruptedBlob)
|
|
||||||
await new BlobStore(historyId0.toString()).putBlob(
|
|
||||||
path,
|
|
||||||
new Blob(gitBlobHash(fileIdBlobExistsInGCSCorrupted), 42)
|
|
||||||
)
|
|
||||||
} finally {
|
|
||||||
await fs.promises.rm(path, { force: true })
|
|
||||||
}
|
|
||||||
await cleanup.postgres()
|
|
||||||
await cleanup.mongo()
|
|
||||||
await Promise.all([
|
|
||||||
testProjects.createEmptyProject(historyId0.toString()),
|
|
||||||
testProjects.createEmptyProject(historyIdDeleted0),
|
|
||||||
])
|
|
||||||
await new BlobStore(historyId0.toString()).putString(
|
|
||||||
fileIdWithDifferentHashNotFound0.toString()
|
|
||||||
)
|
|
||||||
await new BlobStore(historyIdDeleted0.toString()).putString(
|
|
||||||
fileIdWithDifferentHashNotFound1.toString()
|
|
||||||
)
|
|
||||||
await new BlobStore(historyId0.toString()).putString(
|
|
||||||
fileIdInGoodState.toString()
|
|
||||||
)
|
|
||||||
})
|
|
||||||
|
|
||||||
before('populate mongo', async function () {
|
|
||||||
await projectsCollection.insertMany([
|
|
||||||
{
|
|
||||||
_id: projectId0,
|
|
||||||
rootFolder: [
|
|
||||||
{
|
|
||||||
fileRefs: [
|
|
||||||
{ _id: fileIdMissing0 },
|
|
||||||
{ _id: fileIdMissing0 }, // bad file-tree, duplicated fileRef.
|
|
||||||
{ _id: fileIdMissing2 },
|
|
||||||
{ _id: fileIdHashMissing0 },
|
|
||||||
{ _id: fileIdHashMissing1 },
|
|
||||||
{
|
|
||||||
_id: fileIdWithDifferentHashFound,
|
|
||||||
hash: gitBlobHash(fileIdInGoodState),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
_id: fileIdWithDifferentHashRestore,
|
|
||||||
hash: gitBlobHash(fileIdMissing0),
|
|
||||||
},
|
|
||||||
],
|
|
||||||
folders: [
|
|
||||||
{
|
|
||||||
docs: [],
|
|
||||||
},
|
|
||||||
null,
|
|
||||||
{
|
|
||||||
fileRefs: [
|
|
||||||
null,
|
|
||||||
{
|
|
||||||
_id: fileIdInGoodState,
|
|
||||||
hash: gitBlobHash(fileIdInGoodState),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
_id: fileIdWithDifferentHashNotFound0,
|
|
||||||
hash: hashDoesNotExistAsBlob,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
_id: fileIdBlobExistsInGCSCorrupted,
|
|
||||||
hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted),
|
|
||||||
},
|
|
||||||
],
|
|
||||||
folders: [],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
overleaf: { history: { id: historyId0 } },
|
|
||||||
version: 0,
|
|
||||||
},
|
|
||||||
])
|
|
||||||
await deletedProjectsCollection.insertMany([
|
|
||||||
{
|
|
||||||
_id: deleteProjectsRecordId0,
|
|
||||||
project: {
|
|
||||||
_id: projectIdDeleted0,
|
|
||||||
rootFolder: [
|
|
||||||
{
|
|
||||||
fileRefs: [
|
|
||||||
{
|
|
||||||
_id: fileIdWithDifferentHashNotFound1,
|
|
||||||
hash: hashDoesNotExistAsBlob,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
folders: [
|
|
||||||
{
|
|
||||||
fileRefs: [],
|
|
||||||
folders: [
|
|
||||||
{ fileRefs: [{ _id: fileIdMissing1 }], folders: [] },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
overleaf: { history: { id: historyIdDeleted0 } },
|
|
||||||
version: 100,
|
|
||||||
},
|
|
||||||
deleterData: {
|
|
||||||
deletedProjectId: projectIdDeleted0,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
])
|
|
||||||
})
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {Array<string>} args
|
|
||||||
* @param {Record<string, string>} env
|
|
||||||
* @return {Promise<{ stdout: string, stderr: string, status: number }>}
|
|
||||||
*/
|
|
||||||
async function tryRunScript(args = [], env = {}) {
|
|
||||||
let result
|
|
||||||
try {
|
|
||||||
result = await promisify(execFile)(
|
|
||||||
process.argv0,
|
|
||||||
['storage/scripts/back_fill_file_hash_fix_up.mjs', ...args],
|
|
||||||
{
|
|
||||||
encoding: 'utf-8',
|
|
||||||
timeout: TIMEOUT - 500,
|
|
||||||
env: {
|
|
||||||
...process.env,
|
|
||||||
USER_FILES_BUCKET_NAME,
|
|
||||||
SLEEP_BEFORE_EXIT: '1',
|
|
||||||
...env,
|
|
||||||
LOG_LEVEL: 'warn', // Override LOG_LEVEL of acceptance tests
|
|
||||||
},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
result.status = 0
|
|
||||||
} catch (err) {
|
|
||||||
const { stdout, stderr, code } = err
|
|
||||||
if (typeof code !== 'number') {
|
|
||||||
console.log(err)
|
|
||||||
}
|
|
||||||
result = { stdout, stderr, status: code }
|
|
||||||
}
|
|
||||||
expect((await fs.promises.readdir('/tmp')).join(';')).to.not.match(
|
|
||||||
/back_fill_file_hash/
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
async function runScriptWithLogs() {
|
|
||||||
const logsPath = '/tmp/test-script-logs'
|
|
||||||
let result
|
|
||||||
try {
|
|
||||||
await fs.promises.writeFile(
|
|
||||||
logsPath,
|
|
||||||
logs.map(e => JSON.stringify(e)).join('\n')
|
|
||||||
)
|
|
||||||
result = await tryRunScript([`--logs=${logsPath}`])
|
|
||||||
} finally {
|
|
||||||
await fs.promises.rm(logsPath, { force: true })
|
|
||||||
}
|
|
||||||
const stats = JSON.parse(result.stdout.trim().split('\n').pop())
|
|
||||||
return {
|
|
||||||
result,
|
|
||||||
stats,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let result, stats
|
|
||||||
before(async function () {
|
|
||||||
;({ result, stats } = await runScriptWithLogs())
|
|
||||||
})
|
|
||||||
it('should print stats', function () {
|
|
||||||
expect(stats).to.contain({
|
|
||||||
processedLines: 12,
|
|
||||||
success: 7,
|
|
||||||
alreadyProcessed: 0,
|
|
||||||
fileDeleted: 0,
|
|
||||||
skipped: 0,
|
|
||||||
failed: 3,
|
|
||||||
unmatched: 1,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
it('should handle re-run on same logs', async function () {
|
|
||||||
;({ stats } = await runScriptWithLogs())
|
|
||||||
expect(stats).to.contain({
|
|
||||||
processedLines: 12,
|
|
||||||
success: 0,
|
|
||||||
alreadyProcessed: 4,
|
|
||||||
fileDeleted: 3,
|
|
||||||
skipped: 0,
|
|
||||||
failed: 3,
|
|
||||||
unmatched: 1,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
it('should flag the unknown fatal error', function () {
|
|
||||||
const unknown = result.stdout
|
|
||||||
.split('\n')
|
|
||||||
.filter(l => l.includes('unknown fatal error'))
|
|
||||||
expect(unknown).to.have.length(1)
|
|
||||||
const [line] = unknown
|
|
||||||
expect(line).to.exist
|
|
||||||
expect(line).to.include('some other error')
|
|
||||||
})
|
|
||||||
it('should flag the unexpected blob on mismatched hash', function () {
|
|
||||||
const line = result.stdout
|
|
||||||
.split('\n')
|
|
||||||
.find(l => l.includes('found blob with computed filestore object hash'))
|
|
||||||
expect(line).to.exist
|
|
||||||
expect(line).to.include(projectId0.toString())
|
|
||||||
expect(line).to.include(fileIdWithDifferentHashFound.toString())
|
|
||||||
expect(line).to.include(gitBlobHash(fileIdInGoodState))
|
|
||||||
})
|
|
||||||
it('should flag the need to restore', function () {
|
|
||||||
const line = result.stdout
|
|
||||||
.split('\n')
|
|
||||||
.find(l => l.includes('missing blob, need to restore filestore file'))
|
|
||||||
expect(line).to.exist
|
|
||||||
expect(line).to.include(projectId0.toString())
|
|
||||||
expect(line).to.include(fileIdWithDifferentHashRestore.toString())
|
|
||||||
expect(line).to.include(hashDoesNotExistAsBlob)
|
|
||||||
})
|
|
||||||
it('should flag the corrupted blob', function () {
|
|
||||||
const line = result.stdout
|
|
||||||
.split('\n')
|
|
||||||
.find(l => l.includes('blob corrupted'))
|
|
||||||
expect(line).to.exist
|
|
||||||
expect(line).to.include(projectId0.toString())
|
|
||||||
expect(line).to.include(fileIdBlobExistsInGCSCorrupted.toString())
|
|
||||||
expect(line).to.include(
|
|
||||||
gitBlobHashBuffer(Buffer.from(contentCorruptedBlob))
|
|
||||||
)
|
|
||||||
expect(line).to.include(gitBlobHash(fileIdBlobExistsInGCSCorrupted))
|
|
||||||
})
|
|
||||||
it('should update mongo', async function () {
|
|
||||||
expect(await projectsCollection.find({}).toArray())
|
|
||||||
.excludingEvery([
|
|
||||||
'currentEndTimestamp',
|
|
||||||
'currentEndVersion',
|
|
||||||
'updatedAt',
|
|
||||||
'backup',
|
|
||||||
])
|
|
||||||
.to.deep.equal([
|
|
||||||
{
|
|
||||||
_id: projectId0,
|
|
||||||
rootFolder: [
|
|
||||||
{
|
|
||||||
fileRefs: [
|
|
||||||
// Removed
|
|
||||||
// { _id: fileIdMissing0 },
|
|
||||||
// Removed
|
|
||||||
// { _id: fileIdMissing2 },
|
|
||||||
// Added hash
|
|
||||||
{
|
|
||||||
_id: fileIdHashMissing0,
|
|
||||||
hash: gitBlobHash(fileIdHashMissing0),
|
|
||||||
},
|
|
||||||
// Added hash
|
|
||||||
{
|
|
||||||
_id: fileIdHashMissing1,
|
|
||||||
hash: gitBlobHash(fileIdHashMissing1),
|
|
||||||
},
|
|
||||||
// No change, should warn about the find.
|
|
||||||
{
|
|
||||||
_id: fileIdWithDifferentHashFound,
|
|
||||||
hash: gitBlobHash(fileIdInGoodState),
|
|
||||||
},
|
|
||||||
// No change, should warn about the need to restore.
|
|
||||||
{
|
|
||||||
_id: fileIdWithDifferentHashRestore,
|
|
||||||
hash: gitBlobHash(fileIdMissing0),
|
|
||||||
},
|
|
||||||
],
|
|
||||||
folders: [
|
|
||||||
{
|
|
||||||
docs: [],
|
|
||||||
},
|
|
||||||
null,
|
|
||||||
{
|
|
||||||
fileRefs: [
|
|
||||||
null,
|
|
||||||
// No change
|
|
||||||
{
|
|
||||||
_id: fileIdInGoodState,
|
|
||||||
hash: gitBlobHash(fileIdInGoodState),
|
|
||||||
},
|
|
||||||
// Updated hash
|
|
||||||
{
|
|
||||||
_id: fileIdWithDifferentHashNotFound0,
|
|
||||||
hash: gitBlobHash(fileIdWithDifferentHashNotFound0),
|
|
||||||
},
|
|
||||||
// No change, flagged
|
|
||||||
{
|
|
||||||
_id: fileIdBlobExistsInGCSCorrupted,
|
|
||||||
hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted),
|
|
||||||
},
|
|
||||||
],
|
|
||||||
folders: [],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
overleaf: { history: { id: historyId0 } },
|
|
||||||
// Incremented when removing file/updating hash
|
|
||||||
version: 5,
|
|
||||||
},
|
|
||||||
])
|
|
||||||
expect(await deletedProjectsCollection.find({}).toArray()).to.deep.equal([
|
|
||||||
{
|
|
||||||
_id: deleteProjectsRecordId0,
|
|
||||||
project: {
|
|
||||||
_id: projectIdDeleted0,
|
|
||||||
rootFolder: [
|
|
||||||
{
|
|
||||||
fileRefs: [
|
|
||||||
// Updated hash
|
|
||||||
{
|
|
||||||
_id: fileIdWithDifferentHashNotFound1,
|
|
||||||
hash: gitBlobHash(fileIdWithDifferentHashNotFound1),
|
|
||||||
},
|
|
||||||
],
|
|
||||||
folders: [
|
|
||||||
{
|
|
||||||
fileRefs: [],
|
|
||||||
folders: [
|
|
||||||
{
|
|
||||||
fileRefs: [
|
|
||||||
// Removed
|
|
||||||
// { _id: fileIdMissing1 },
|
|
||||||
],
|
|
||||||
folders: [],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
overleaf: { history: { id: historyIdDeleted0 } },
|
|
||||||
// Incremented when removing file/updating hash
|
|
||||||
version: 102,
|
|
||||||
},
|
|
||||||
deleterData: {
|
|
||||||
deletedProjectId: projectIdDeleted0,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
])
|
|
||||||
const writtenBlobsByProject = new Map()
|
|
||||||
for (const { projectId, fileId } of writtenBlobs) {
|
|
||||||
writtenBlobsByProject.set(
|
|
||||||
projectId,
|
|
||||||
(writtenBlobsByProject.get(projectId) || []).concat([fileId])
|
|
||||||
)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
it('should have written the back filled files to history v1', async function () {
|
|
||||||
for (const { historyId, fileId } of writtenBlobs) {
|
|
||||||
const blobStore = new BlobStore(historyId.toString())
|
|
||||||
const hash = gitBlobHash(fileId.toString())
|
|
||||||
const blob = await blobStore.getBlob(hash)
|
|
||||||
expect(blob).to.exist
|
|
||||||
expect(blob.getByteLength()).to.equal(24)
|
|
||||||
const id = await blobStore.getString(hash)
|
|
||||||
expect(id).to.equal(fileId.toString())
|
|
||||||
// double check we are not comparing 'undefined' or '[object Object]' above
|
|
||||||
expect(id).to.match(/^[a-f0-9]{24}$/)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
Reference in New Issue
Block a user