mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-12 13:13:45 +01:00
[e2e] cut down flakeyness (#35896)
* [saas-e2e] conversion: wait for initial compile before looking at PDF * [eslint-plugin] s/before/beforeWithReRunOnTestRetry/ in e2e tests * [server-ce] remove filestore migration scripts and E2E tests * [server-ce] test: trim down upgrading tests to 5.x -> latest GitOrigin-RevId: 2426ec4a4f3d67a5e0e7850ea84a942ae4619f94
This commit is contained in:
Jakob Ackermann
@@ -22,5 +22,6 @@ module.exports = {
|
||||
'sorted-keys-in-locales': require('./sorted-keys-in-locales'),
|
||||
'locale-variables-match-en': require('./locale-variables-match-en'),
|
||||
'no-orphan-locale-keys': require('./no-orphan-locale-keys'),
|
||||
'no-mocha-before': require('./no-mocha-before'),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
const Path = require('node:path')
|
||||
|
||||
/**
|
||||
* Block bare mocha `before()` hooks in Cypress e2e suites.
|
||||
*
|
||||
* `before()` hooks run once per suite and do NOT re-run when a test retries,
|
||||
* while `beforeEach`-based hooks (like `beforeWithReRunOnTestRetry`) do.
|
||||
* Mixing the two also changes ordering: all `before()` hooks run ahead of
|
||||
* every `beforeEach`, not in source order. This matters in the Server Pro
|
||||
* e2e suite, where suites change the server config via `startWith()` while
|
||||
* test setup runs in `beforeWithReRunOnTestRetry()`: when a later suite
|
||||
* changes the config, both must re-run on retry and in registration order,
|
||||
* otherwise a retried test runs against setup or config belonging to a
|
||||
* different suite.
|
||||
*
|
||||
* The autofix rewrites `before(fn)` to `beforeWithReRunOnTestRetry(fn)` and
|
||||
* adds the helper import when missing. Configure the helper location (repo
|
||||
* root relative, no extension) via the `helperPath` option.
|
||||
*/
|
||||
module.exports = {
|
||||
meta: {
|
||||
type: 'suggestion',
|
||||
docs: {
|
||||
description:
|
||||
'Disallow mocha before() hooks in favor of beforeWithReRunOnTestRetry()',
|
||||
},
|
||||
fixable: 'code',
|
||||
schema: [
|
||||
{
|
||||
type: 'object',
|
||||
properties: {
|
||||
helperPath: { type: 'string' },
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
],
|
||||
messages: {
|
||||
noMochaBefore:
|
||||
'before() hooks do not re-run when a test retries, so retried tests run against stale setup. Use beforeWithReRunOnTestRetry() instead.',
|
||||
},
|
||||
},
|
||||
create(context) {
|
||||
const helperPath = context.options[0]?.helperPath
|
||||
let lastImport = null
|
||||
let hasHelperImport = false
|
||||
|
||||
function helperImportSpecifier() {
|
||||
const absolute = Path.resolve(context.cwd, helperPath)
|
||||
let relative = Path.relative(
|
||||
Path.dirname(Path.resolve(context.cwd, context.filename)),
|
||||
absolute
|
||||
)
|
||||
relative = relative.split(Path.sep).join('/')
|
||||
if (!relative.startsWith('.')) {
|
||||
relative = `./${relative}`
|
||||
}
|
||||
return relative
|
||||
}
|
||||
|
||||
return {
|
||||
ImportDeclaration(node) {
|
||||
lastImport = node
|
||||
if (
|
||||
node.specifiers.some(
|
||||
specifier =>
|
||||
specifier.type === 'ImportSpecifier' &&
|
||||
specifier.imported.name === 'beforeWithReRunOnTestRetry'
|
||||
)
|
||||
) {
|
||||
hasHelperImport = true
|
||||
}
|
||||
},
|
||||
CallExpression(node) {
|
||||
if (
|
||||
node.callee.type !== 'Identifier' ||
|
||||
node.callee.name !== 'before'
|
||||
) {
|
||||
return
|
||||
}
|
||||
// Only the global mocha hook; skip locally defined/imported `before`
|
||||
const variable = context.sourceCode
|
||||
.getScope(node)
|
||||
.references.find(ref => ref.identifier === node.callee)?.resolved
|
||||
if (variable && variable.defs.length > 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const fixable = node.arguments.length === 1 && helperPath
|
||||
context.report({
|
||||
node: node.callee,
|
||||
messageId: 'noMochaBefore',
|
||||
fix: fixable
|
||||
? fixer => {
|
||||
const fixes = [
|
||||
fixer.replaceText(node.callee, 'beforeWithReRunOnTestRetry'),
|
||||
]
|
||||
if (!hasHelperImport) {
|
||||
const importText = `import { beforeWithReRunOnTestRetry } from '${helperImportSpecifier()}'`
|
||||
fixes.push(
|
||||
lastImport
|
||||
? fixer.insertTextAfter(lastImport, `\n${importText}`)
|
||||
: fixer.insertTextBeforeRange([0, 0], `${importText}\n`)
|
||||
)
|
||||
}
|
||||
return fixes
|
||||
}
|
||||
: undefined,
|
||||
})
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import noConsecutiveSpacesInLocales from '../../no-consecutive-spaces-in-locales
|
||||
import noStraightApostrophesInLocales from '../../no-straight-apostrophes-in-locales.js'
|
||||
import frenchTypographyInLocales from '../../french-typography-in-locales.js'
|
||||
import sortedKeysInLocales from '../../sorted-keys-in-locales.js'
|
||||
import noMochaBefore from '../../no-mocha-before.js'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
|
||||
@@ -546,3 +547,81 @@ jsonRuleTester.run('french-typography-in-locales', frenchTypographyInLocales, {
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const noMochaBeforeOptions = [
|
||||
{ helperPath: 'server-ce/test/helpers/beforeWithReRunOnTestRetry' },
|
||||
]
|
||||
|
||||
ruleTester.run('no-mocha-before', noMochaBefore, {
|
||||
valid: [
|
||||
{
|
||||
code: `beforeEach(function () { setup() })`,
|
||||
options: noMochaBeforeOptions,
|
||||
},
|
||||
{
|
||||
code: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
|
||||
beforeWithReRunOnTestRetry(function () { setup() })`,
|
||||
options: noMochaBeforeOptions,
|
||||
},
|
||||
{
|
||||
// locally defined `before` is not the mocha hook
|
||||
code: `function before(fn) { fn() }
|
||||
before(function () { setup() })`,
|
||||
options: noMochaBeforeOptions,
|
||||
},
|
||||
],
|
||||
invalid: [
|
||||
{
|
||||
// rename + import added after the last existing import
|
||||
filename: 'server-ce/test/foo.spec.ts',
|
||||
code: `import { login } from './helpers/login'
|
||||
before(async function () { this.timeout(1000) })`,
|
||||
options: noMochaBeforeOptions,
|
||||
errors: 1,
|
||||
output: `import { login } from './helpers/login'
|
||||
import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
|
||||
beforeWithReRunOnTestRetry(async function () { this.timeout(1000) })`,
|
||||
},
|
||||
{
|
||||
// no imports yet: import inserted at the top
|
||||
filename: 'server-ce/test/foo.spec.ts',
|
||||
code: `before(function () { setup() })`,
|
||||
options: noMochaBeforeOptions,
|
||||
errors: 1,
|
||||
output: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
|
||||
beforeWithReRunOnTestRetry(function () { setup() })`,
|
||||
},
|
||||
{
|
||||
// helper already imported: rename only
|
||||
filename: 'server-ce/test/foo.spec.ts',
|
||||
code: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
|
||||
before(function () { setup() })`,
|
||||
options: noMochaBeforeOptions,
|
||||
errors: 1,
|
||||
output: `import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
|
||||
beforeWithReRunOnTestRetry(function () { setup() })`,
|
||||
},
|
||||
{
|
||||
// file inside helpers/: relative path has no directory prefix
|
||||
filename: 'server-ce/test/helpers/config.ts',
|
||||
code: `before(function () { setup() })`,
|
||||
options: noMochaBeforeOptions,
|
||||
errors: 1,
|
||||
output: `import { beforeWithReRunOnTestRetry } from './beforeWithReRunOnTestRetry'
|
||||
beforeWithReRunOnTestRetry(function () { setup() })`,
|
||||
},
|
||||
{
|
||||
// two-argument mocha form: report without autofix
|
||||
filename: 'server-ce/test/foo.spec.ts',
|
||||
code: `before('named hook', function () { setup() })`,
|
||||
options: noMochaBeforeOptions,
|
||||
errors: 1,
|
||||
},
|
||||
{
|
||||
// no helperPath configured: report without autofix
|
||||
filename: 'server-ce/test/foo.spec.ts',
|
||||
code: `before(function () { setup() })`,
|
||||
errors: 1,
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,629 +0,0 @@
|
||||
// @ts-check
|
||||
import Events from 'node:events'
|
||||
import fs from 'node:fs'
|
||||
import Stream from 'node:stream'
|
||||
import { ObjectId } from 'mongodb'
|
||||
import logger from '@overleaf/logger'
|
||||
import OError from '@overleaf/o-error'
|
||||
import { Blob } from 'overleaf-editor-core'
|
||||
import {
|
||||
BlobStore,
|
||||
getStringLengthOfFile,
|
||||
makeBlobForFile,
|
||||
} from '../lib/blob_store/index.js'
|
||||
import { db } from '../lib/mongodb.js'
|
||||
import commandLineArgs from 'command-line-args'
|
||||
import readline from 'node:readline'
|
||||
import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
|
||||
import { setTimeout } from 'node:timers/promises'
|
||||
|
||||
// Silence warning.
|
||||
Events.setMaxListeners(20)
|
||||
|
||||
// Enable caching for ObjectId.toString()
|
||||
ObjectId.cacheHexString = true
|
||||
|
||||
/**
|
||||
* @typedef {import("mongodb").Collection} Collection
|
||||
* @typedef {import("mongodb").Collection<Project>} ProjectsCollection
|
||||
* @typedef {import("mongodb").Collection<{project: Project}>} DeletedProjectsCollection
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} FileRef
|
||||
* @property {ObjectId} _id
|
||||
* @property {string} hash
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} Folder
|
||||
* @property {Array<Folder>} folders
|
||||
* @property {Array<FileRef>} fileRefs
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} Project
|
||||
* @property {ObjectId} _id
|
||||
* @property {Array<Folder>} rootFolder
|
||||
* @property {{history: {id: (number|string)}}} overleaf
|
||||
*/
|
||||
|
||||
/**
|
||||
* @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_MISSING_HASH: boolean, LOGS: string}}
|
||||
*/
|
||||
function parseArgs() {
|
||||
const args = commandLineArgs([
|
||||
{ name: 'fixNotFound', type: String, defaultValue: 'true' },
|
||||
{ name: 'fixHashMismatch', type: String, defaultValue: 'true' },
|
||||
{ name: 'fixMissingHash', type: String, defaultValue: 'true' },
|
||||
{ name: 'logs', type: String, defaultValue: '' },
|
||||
])
|
||||
/**
|
||||
* commandLineArgs cannot handle --foo=false, so go the long way
|
||||
* @param {string} name
|
||||
* @return {boolean}
|
||||
*/
|
||||
function boolVal(name) {
|
||||
const v = args[name]
|
||||
if (['true', 'false'].includes(v)) return v === 'true'
|
||||
throw new Error(`expected "true" or "false" for boolean option ${name}`)
|
||||
}
|
||||
return {
|
||||
FIX_HASH_MISMATCH: boolVal('fixNotFound'),
|
||||
FIX_NOT_FOUND: boolVal('fixHashMismatch'),
|
||||
FIX_MISSING_HASH: boolVal('fixMissingHash'),
|
||||
LOGS: args.logs,
|
||||
}
|
||||
}
|
||||
|
||||
const { FIX_HASH_MISMATCH, FIX_NOT_FOUND, FIX_MISSING_HASH, LOGS } = parseArgs()
|
||||
if (!LOGS) {
|
||||
throw new Error('--logs parameter missing')
|
||||
}
|
||||
const BUFFER_DIR = fs.mkdtempSync(
|
||||
process.env.BUFFER_DIR_PREFIX || '/tmp/back_fill_file_hash-'
|
||||
)
|
||||
const USER_FILES_BUCKET_NAME = process.env.USER_FILES_BUCKET_NAME || ''
|
||||
if (!USER_FILES_BUCKET_NAME) {
|
||||
throw new Error('env var USER_FILES_BUCKET_NAME is missing')
|
||||
}
|
||||
// https://nodejs.org/api/stream.html#streamgetdefaulthighwatermarkobjectmode
|
||||
const STREAM_HIGH_WATER_MARK = parseInt(
|
||||
process.env.STREAM_HIGH_WATER_MARK || (64 * 1024).toString(),
|
||||
10
|
||||
)
|
||||
const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10)
|
||||
|
||||
// Filestore endpoint location
|
||||
const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1'
|
||||
const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009'
|
||||
|
||||
async function fetchFromFilestore(projectId, fileId) {
|
||||
const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}`
|
||||
const response = await fetch(url)
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError('file not found in filestore', {
|
||||
status: response.status,
|
||||
})
|
||||
}
|
||||
const body = await response.text()
|
||||
throw new OError('fetchFromFilestore failed', {
|
||||
projectId,
|
||||
fileId,
|
||||
status: response.status,
|
||||
body,
|
||||
})
|
||||
}
|
||||
if (!response.body) {
|
||||
throw new OError('fetchFromFilestore response has no body', {
|
||||
projectId,
|
||||
fileId,
|
||||
status: response.status,
|
||||
})
|
||||
}
|
||||
return response.body
|
||||
}
|
||||
|
||||
/** @type {ProjectsCollection} */
|
||||
const projectsCollection = db.collection('projects')
|
||||
/** @type {DeletedProjectsCollection} */
|
||||
const deletedProjectsCollection = db.collection('deletedProjects')
|
||||
|
||||
let gracefulShutdownInitiated = false
|
||||
|
||||
process.on('SIGINT', handleSignal)
|
||||
process.on('SIGTERM', handleSignal)
|
||||
|
||||
function handleSignal() {
|
||||
gracefulShutdownInitiated = true
|
||||
console.warn('graceful shutdown initiated, draining queue')
|
||||
}
|
||||
|
||||
class FileDeletedError extends OError {}
|
||||
|
||||
/** @type {Map<string,{project: Project, projectSoftDeleted: boolean}>} */
|
||||
const PROJECT_CACHE = new Map()
|
||||
|
||||
/**
|
||||
* @param {string} projectId
|
||||
* @return {Promise<{project: Project, projectSoftDeleted: boolean}>}
|
||||
*/
|
||||
async function getProject(projectId) {
|
||||
const cached = PROJECT_CACHE.get(projectId)
|
||||
if (cached) return cached
|
||||
|
||||
let projectSoftDeleted
|
||||
let project = await projectsCollection.findOne({
|
||||
_id: new ObjectId(projectId),
|
||||
})
|
||||
if (project) {
|
||||
projectSoftDeleted = false
|
||||
} else {
|
||||
const softDeleted = await deletedProjectsCollection.findOne({
|
||||
'deleterData.deletedProjectId': new ObjectId(projectId),
|
||||
project: { $exists: true },
|
||||
})
|
||||
if (!softDeleted) {
|
||||
throw new OError('project hard-deleted')
|
||||
}
|
||||
project = softDeleted.project
|
||||
projectSoftDeleted = true
|
||||
}
|
||||
PROJECT_CACHE.set(projectId, { projectSoftDeleted, project })
|
||||
return { projectSoftDeleted, project }
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Folder} folder
|
||||
* @param {string} fileId
|
||||
* @return {{path: string, fileRef: FileRef, folder: Folder}|null}
|
||||
*/
|
||||
function getFileTreePath(folder, fileId) {
|
||||
if (!folder) return null
|
||||
let idx = 0
|
||||
if (Array.isArray(folder.fileRefs)) {
|
||||
for (const fileRef of folder.fileRefs) {
|
||||
if (fileRef?._id.toString() === fileId) {
|
||||
return {
|
||||
fileRef,
|
||||
path: `.fileRefs.${idx}`,
|
||||
folder,
|
||||
}
|
||||
}
|
||||
idx++
|
||||
}
|
||||
}
|
||||
idx = 0
|
||||
if (Array.isArray(folder.folders)) {
|
||||
for (const child of folder.folders) {
|
||||
const match = getFileTreePath(child, fileId)
|
||||
if (match) {
|
||||
return {
|
||||
fileRef: match.fileRef,
|
||||
folder: match.folder,
|
||||
path: `.folders.${idx}${match.path}`,
|
||||
}
|
||||
}
|
||||
idx++
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} projectId
|
||||
* @param {string} fileId
|
||||
* @return {Promise<{fileRef: FileRef, folder: Folder, fullPath: string, query: Object, projectSoftDeleted: boolean}>}
|
||||
*/
|
||||
async function findFile(projectId, fileId) {
|
||||
const { projectSoftDeleted, project } = await getProject(projectId)
|
||||
const match = getFileTreePath(project.rootFolder[0], fileId)
|
||||
if (!match) {
|
||||
throw new FileDeletedError('file not found in file-tree', {
|
||||
projectSoftDeleted,
|
||||
})
|
||||
}
|
||||
const { path, fileRef, folder } = match
|
||||
let fullPath
|
||||
let query
|
||||
if (projectSoftDeleted) {
|
||||
fullPath = `project.rootFolder.0${path}`
|
||||
query = {
|
||||
'deleterData.deletedProjectId': new ObjectId(projectId),
|
||||
[`${fullPath}._id`]: new ObjectId(fileId),
|
||||
}
|
||||
} else {
|
||||
fullPath = `rootFolder.0${path}`
|
||||
query = {
|
||||
_id: new ObjectId(projectId),
|
||||
[`${fullPath}._id`]: new ObjectId(fileId),
|
||||
}
|
||||
}
|
||||
return {
|
||||
projectSoftDeleted,
|
||||
query,
|
||||
fullPath,
|
||||
fileRef,
|
||||
folder,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} line
|
||||
* @return {Promise<boolean>}
|
||||
*/
|
||||
async function fixNotFound(line) {
|
||||
const { projectId, fileId, bucketName } = JSON.parse(line)
|
||||
if (bucketName !== USER_FILES_BUCKET_NAME) {
|
||||
throw new OError('not found case for another bucket')
|
||||
}
|
||||
|
||||
const { projectSoftDeleted, query, fullPath, fileRef, folder } =
|
||||
await findFile(projectId, fileId)
|
||||
logger.info({ projectId, fileId, fileRef }, 'removing fileRef')
|
||||
// Copied from _removeElementFromMongoArray (https://github.com/overleaf/internal/blob/11e09528c153de6b7766d18c3c90d94962190371/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js)
|
||||
const nonArrayPath = fullPath.slice(0, fullPath.lastIndexOf('.'))
|
||||
let result
|
||||
if (projectSoftDeleted) {
|
||||
result = await deletedProjectsCollection.updateOne(query, {
|
||||
$pull: { [nonArrayPath]: { _id: new ObjectId(fileId) } },
|
||||
$inc: { 'project.version': 1 },
|
||||
})
|
||||
} else {
|
||||
result = await projectsCollection.updateOne(query, {
|
||||
$pull: { [nonArrayPath]: { _id: new ObjectId(fileId) } },
|
||||
$inc: { version: 1 },
|
||||
})
|
||||
}
|
||||
if (result.matchedCount !== 1) {
|
||||
throw new OError('file-tree write did not match', { result })
|
||||
}
|
||||
// Update the cache. The mongo-path of the next file will be off otherwise.
|
||||
folder.fileRefs = folder.fileRefs.filter(f => !f._id.equals(fileId))
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} projectId
|
||||
* @param {string} fileId
|
||||
* @param {string} hash
|
||||
* @return {Promise<void>}
|
||||
*/
|
||||
async function setHashInMongo(projectId, fileId, hash) {
|
||||
const { projectSoftDeleted, query, fullPath, fileRef } = await findFile(
|
||||
projectId,
|
||||
fileId
|
||||
)
|
||||
if (fileRef.hash === hash) return
|
||||
logger.info({ projectId, fileId, fileRef, hash }, 'setting fileRef hash')
|
||||
let result
|
||||
if (projectSoftDeleted) {
|
||||
result = await deletedProjectsCollection.updateOne(query, {
|
||||
$set: { [`${fullPath}.hash`]: hash },
|
||||
$inc: { 'project.version': 1 },
|
||||
})
|
||||
} else {
|
||||
result = await projectsCollection.updateOne(query, {
|
||||
$set: { [`${fullPath}.hash`]: hash },
|
||||
$inc: { version: 1 },
|
||||
})
|
||||
}
|
||||
if (result.matchedCount !== 1) {
|
||||
throw new OError('file-tree write did not match', { result })
|
||||
}
|
||||
fileRef.hash = hash // Update cache for completeness.
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} projectId
|
||||
* @param {string} fileId
|
||||
* @param {string} historyId
|
||||
* @return {Promise<void>}
|
||||
*/
|
||||
async function importRestoredFilestoreFile(projectId, fileId, historyId) {
|
||||
const path = `${BUFFER_DIR}/${projectId}_${fileId}`
|
||||
try {
|
||||
let s
|
||||
try {
|
||||
s = await fetchFromFilestore(projectId, fileId)
|
||||
} catch (err) {
|
||||
if (err instanceof NotFoundError) {
|
||||
throw new OError('missing blob, need to restore filestore file', {
|
||||
projectId,
|
||||
fileId,
|
||||
})
|
||||
}
|
||||
throw err
|
||||
}
|
||||
await Stream.promises.pipeline(
|
||||
s,
|
||||
fs.createWriteStream(path, { highWaterMark: STREAM_HIGH_WATER_MARK })
|
||||
)
|
||||
const blobStore = new BlobStore(historyId)
|
||||
const blob = await blobStore.putFile(path)
|
||||
await setHashInMongo(projectId, fileId, blob.getHash())
|
||||
} finally {
|
||||
await fs.promises.rm(path, { force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} projectId
|
||||
* @param {string} fileId
|
||||
* @param {string} path
|
||||
* @return {Promise<Blob>}
|
||||
*/
|
||||
async function bufferFilestoreFileToDisk(projectId, fileId, path) {
|
||||
try {
|
||||
await Stream.promises.pipeline(
|
||||
await fetchFromFilestore(projectId, fileId),
|
||||
fs.createWriteStream(path, { highWaterMark: STREAM_HIGH_WATER_MARK })
|
||||
)
|
||||
const blob = await makeBlobForFile(path)
|
||||
blob.setStringLength(
|
||||
await getStringLengthOfFile(blob.getByteLength(), path)
|
||||
)
|
||||
return blob
|
||||
} catch (err) {
|
||||
if (err instanceof NotFoundError) {
|
||||
throw new OError('missing blob, need to restore filestore file', {
|
||||
projectId,
|
||||
fileId,
|
||||
})
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} projectId
|
||||
* @param {string} fileId
|
||||
* @return {Promise<string>}
|
||||
*/
|
||||
async function computeFilestoreFileHash(projectId, fileId) {
|
||||
const path = `${BUFFER_DIR}/${projectId}_${fileId}`
|
||||
try {
|
||||
const blob = await bufferFilestoreFileToDisk(projectId, fileId, path)
|
||||
return blob.getHash()
|
||||
} finally {
|
||||
await fs.promises.rm(path, { force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} projectId
|
||||
* @param {string} fileId
|
||||
* @return {Promise<void>}
|
||||
*/
|
||||
async function uploadFilestoreFile(projectId, fileId) {
|
||||
const path = `${BUFFER_DIR}/${projectId}_${fileId}`
|
||||
try {
|
||||
const blob = await bufferFilestoreFileToDisk(projectId, fileId, path)
|
||||
const hash = blob.getHash()
|
||||
try {
|
||||
await ensureBlobExistsForFile(projectId, fileId, hash)
|
||||
} catch (err) {
|
||||
if (!(err instanceof Blob.NotFoundError)) throw err
|
||||
|
||||
const { project } = await getProject(projectId)
|
||||
const historyId = project.overleaf.history.id.toString()
|
||||
const blobStore = new BlobStore(historyId)
|
||||
await blobStore.putBlob(path, blob)
|
||||
await ensureBlobExistsForFile(projectId, fileId, hash)
|
||||
}
|
||||
} finally {
|
||||
await fs.promises.rm(path, { force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} line
|
||||
* @return {Promise<boolean>}
|
||||
*/
|
||||
async function fixHashMismatch(line) {
|
||||
const {
|
||||
projectId,
|
||||
fileId,
|
||||
hash: computedHash,
|
||||
entry: {
|
||||
hash: fileTreeHash,
|
||||
ctx: { historyId },
|
||||
},
|
||||
} = JSON.parse(line)
|
||||
const blobStore = new BlobStore(historyId)
|
||||
if (await blobStore.getBlob(fileTreeHash)) {
|
||||
throw new OError('found blob with computed filestore object hash')
|
||||
}
|
||||
if (!(await blobStore.getBlob(computedHash))) {
|
||||
await importRestoredFilestoreFile(projectId, fileId, historyId)
|
||||
return true
|
||||
}
|
||||
return await ensureBlobExistsForFile(projectId, fileId, computedHash)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} projectId
|
||||
* @param {string} fileId
|
||||
* @param {string} hash
|
||||
* @return {Promise<boolean>}
|
||||
*/
|
||||
async function hashAlreadyUpdatedInFileTree(projectId, fileId, hash) {
|
||||
const { fileRef } = await findFile(projectId, fileId)
|
||||
return fileRef.hash === hash
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} projectId
|
||||
* @param {string} fileId
|
||||
* @param {string} hash
|
||||
* @return {Promise<boolean>}
|
||||
*/
|
||||
async function ensureBlobExistsForFile(projectId, fileId, hash) {
|
||||
const { project } = await getProject(projectId)
|
||||
const historyId = project.overleaf.history.id.toString()
|
||||
const blobStore = new BlobStore(historyId)
|
||||
if (
|
||||
(await hashAlreadyUpdatedInFileTree(projectId, fileId, hash)) &&
|
||||
(await blobStore.getBlob(hash))
|
||||
) {
|
||||
return false // already processed
|
||||
}
|
||||
|
||||
const stream = await blobStore.getStream(hash)
|
||||
const path = `${BUFFER_DIR}/${historyId}_${hash}`
|
||||
try {
|
||||
await Stream.promises.pipeline(
|
||||
stream,
|
||||
fs.createWriteStream(path, {
|
||||
highWaterMark: STREAM_HIGH_WATER_MARK,
|
||||
})
|
||||
)
|
||||
|
||||
const writtenBlob = await makeBlobForFile(path)
|
||||
writtenBlob.setStringLength(
|
||||
await getStringLengthOfFile(writtenBlob.getByteLength(), path)
|
||||
)
|
||||
if (writtenBlob.getHash() !== hash) {
|
||||
// Double check download, better safe than sorry.
|
||||
throw new OError('blob corrupted', { writtenBlob, hash })
|
||||
}
|
||||
|
||||
let blob = await blobStore.getBlob(hash)
|
||||
if (!blob) {
|
||||
// Calling blobStore.putBlob would result in the same error again.
|
||||
// HACK: Skip upload to GCS and finalize putBlob operation directly.
|
||||
await blobStore.backend.insertBlob(historyId, writtenBlob)
|
||||
}
|
||||
} finally {
|
||||
await fs.promises.rm(path, { force: true })
|
||||
}
|
||||
await setHashInMongo(projectId, fileId, hash)
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} line
|
||||
* @return {Promise<boolean>}
|
||||
*/
|
||||
async function fixMissingHash(line) {
|
||||
let { projectId, _id: fileId } = JSON.parse(line)
|
||||
const {
|
||||
fileRef: { hash },
|
||||
} = await findFile(projectId, fileId)
|
||||
if (hash) {
|
||||
// processed, double check
|
||||
return await ensureBlobExistsForFile(projectId, fileId, hash)
|
||||
}
|
||||
await uploadFilestoreFile(projectId, fileId)
|
||||
return true
|
||||
}
|
||||
|
||||
const CASES = {
|
||||
'not found': {
|
||||
match: 'NotFoundError',
|
||||
flag: FIX_NOT_FOUND,
|
||||
action: fixNotFound,
|
||||
},
|
||||
'hash mismatch': {
|
||||
match: 'OError: hash mismatch',
|
||||
flag: FIX_HASH_MISMATCH,
|
||||
action: fixHashMismatch,
|
||||
},
|
||||
'missing file hash': {
|
||||
match: '"bad file hash"',
|
||||
flag: FIX_MISSING_HASH,
|
||||
action: fixMissingHash,
|
||||
},
|
||||
}
|
||||
|
||||
const STATS = {
|
||||
processedLines: 0,
|
||||
success: 0,
|
||||
alreadyProcessed: 0,
|
||||
fileDeleted: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
unmatched: 0,
|
||||
}
|
||||
function logStats() {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
time: new Date(),
|
||||
gracefulShutdownInitiated,
|
||||
...STATS,
|
||||
})
|
||||
)
|
||||
}
|
||||
setInterval(logStats, 10_000)
|
||||
|
||||
async function processLog() {
|
||||
const rl = readline.createInterface({
|
||||
input: fs.createReadStream(LOGS),
|
||||
})
|
||||
nextLine: for await (const line of rl) {
|
||||
if (gracefulShutdownInitiated) break
|
||||
STATS.processedLines++
|
||||
if (
|
||||
!(
|
||||
line.includes('"failed to process file"') ||
|
||||
// Process missing hashes as flagged by find_malformed_filetrees.mjs
|
||||
line.includes('"bad file-tree path"')
|
||||
)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
for (const [name, { match, flag, action }] of Object.entries(CASES)) {
|
||||
if (!line.includes(match)) continue
|
||||
if (flag) {
|
||||
try {
|
||||
if (await action(line)) {
|
||||
STATS.success++
|
||||
} else {
|
||||
STATS.alreadyProcessed++
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof FileDeletedError) {
|
||||
STATS.fileDeleted++
|
||||
logger.info({ err, line }, 'file deleted, skipping')
|
||||
} else {
|
||||
STATS.failed++
|
||||
logger.error({ err, line }, `failed to fix ${name}`)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
STATS.skipped++
|
||||
}
|
||||
continue nextLine
|
||||
}
|
||||
STATS.unmatched++
|
||||
logger.warn({ line }, 'unknown fatal error')
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
await processLog()
|
||||
} finally {
|
||||
logStats()
|
||||
try {
|
||||
await fs.promises.rm(BUFFER_DIR, { recursive: true, force: true })
|
||||
} catch (err) {
|
||||
console.error(`Cleanup of BUFFER_DIR=${BUFFER_DIR} failed`, err)
|
||||
}
|
||||
}
|
||||
const { skipped, failed, unmatched } = STATS
|
||||
await setTimeout(SLEEP_BEFORE_EXIT)
|
||||
if (failed > 0) {
|
||||
process.exit(Math.min(failed, 99))
|
||||
} else if (unmatched > 0) {
|
||||
process.exit(100)
|
||||
} else if (skipped > 0) {
|
||||
process.exit(101)
|
||||
} else {
|
||||
process.exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
await main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,621 +0,0 @@
|
||||
import fs from 'node:fs'
|
||||
import Crypto from 'node:crypto'
|
||||
import { promisify } from 'node:util'
|
||||
import { Binary, ObjectId } from 'mongodb'
|
||||
import { Blob } from 'overleaf-editor-core'
|
||||
import { db } from '../../../../storage/lib/mongodb.js'
|
||||
import cleanup from './support/cleanup.js'
|
||||
import testProjects from '../api/support/test_projects.js'
|
||||
import { execFile } from 'node:child_process'
|
||||
import chai, { expect } from 'chai'
|
||||
import chaiExclude from 'chai-exclude'
|
||||
import { BlobStore } from '../../../../storage/lib/blob_store/index.js'
|
||||
import { mockFilestore } from './support/MockFilestore.mjs'
|
||||
|
||||
chai.use(chaiExclude)
|
||||
|
||||
const TIMEOUT = 20 * 1_000
|
||||
|
||||
const projectsCollection = db.collection('projects')
|
||||
const deletedProjectsCollection = db.collection('deletedProjects')
|
||||
|
||||
/**
|
||||
* @param {ObjectId} objectId
|
||||
* @return {string}
|
||||
*/
|
||||
function gitBlobHash(objectId) {
|
||||
return gitBlobHashBuffer(Buffer.from(objectId.toString()))
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Buffer} buf
|
||||
* @return {string}
|
||||
*/
|
||||
function gitBlobHashBuffer(buf) {
|
||||
const sha = Crypto.createHash('sha1')
|
||||
sha.update(`blob ${buf.byteLength}\x00`)
|
||||
sha.update(buf)
|
||||
return sha.digest('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} gitBlobHash
|
||||
* @return {Binary}
|
||||
*/
|
||||
function binaryForGitBlobHash(gitBlobHash) {
|
||||
return new Binary(Buffer.from(gitBlobHash, 'hex'))
|
||||
}
|
||||
|
||||
function objectIdFromTime(timestamp) {
|
||||
return ObjectId.createFromTime(new Date(timestamp).getTime() / 1000)
|
||||
}
|
||||
|
||||
const PRINT_IDS_AND_HASHES_FOR_DEBUGGING = false
|
||||
|
||||
describe('back_fill_file_hash_fix_up script', function () {
|
||||
this.timeout(TIMEOUT)
|
||||
const USER_FILES_BUCKET_NAME = 'fake-user-files-gcs'
|
||||
|
||||
const projectId0 = objectIdFromTime('2017-01-01T00:00:00Z')
|
||||
const projectIdDeleted0 = objectIdFromTime('2017-01-01T00:04:00Z')
|
||||
const historyId0 = 42 // stored as number is mongo
|
||||
const historyIdDeleted0 = projectIdDeleted0.toString()
|
||||
const fileIdWithDifferentHashFound = objectIdFromTime('2017-02-01T00:00:00Z')
|
||||
const fileIdInGoodState = objectIdFromTime('2017-02-01T00:01:00Z')
|
||||
const fileIdWithDifferentHashNotFound0 = objectIdFromTime(
|
||||
'2017-02-01T00:03:00Z'
|
||||
)
|
||||
const fileIdWithDifferentHashNotFound1 = objectIdFromTime(
|
||||
'2017-02-01T00:04:00Z'
|
||||
)
|
||||
const fileIdBlobExistsInGCSCorrupted = objectIdFromTime(
|
||||
'2017-02-01T00:05:00Z'
|
||||
)
|
||||
const fileIdMissing0 = objectIdFromTime('2024-02-01T00:06:00Z')
|
||||
const fileIdMissing1 = objectIdFromTime('2017-02-01T00:07:00Z')
|
||||
const fileIdWithDifferentHashRestore = objectIdFromTime(
|
||||
'2017-02-01T00:08:00Z'
|
||||
)
|
||||
const fileIdMissing2 = objectIdFromTime('2017-02-01T00:12:00Z')
|
||||
const fileIdHashMissing0 = objectIdFromTime('2017-02-01T00:13:00Z')
|
||||
const fileIdHashMissing1 = objectIdFromTime('2017-02-01T00:14:00Z')
|
||||
const contentCorruptedBlob = 'string that produces another hash'
|
||||
const contentDoesNotExistAsBlob = 'does not exist as blob'
|
||||
const hashDoesNotExistAsBlob = gitBlobHashBuffer(
|
||||
Buffer.from(contentDoesNotExistAsBlob)
|
||||
)
|
||||
const deleteProjectsRecordId0 = new ObjectId()
|
||||
const writtenBlobs = [
|
||||
{
|
||||
projectId: projectId0,
|
||||
historyId: historyId0,
|
||||
fileId: fileIdWithDifferentHashNotFound0,
|
||||
},
|
||||
{
|
||||
projectId: projectId0,
|
||||
historyId: historyId0,
|
||||
fileId: fileIdHashMissing0,
|
||||
},
|
||||
{
|
||||
projectId: projectId0,
|
||||
historyId: historyId0,
|
||||
fileId: fileIdHashMissing1,
|
||||
},
|
||||
{
|
||||
projectId: projectIdDeleted0,
|
||||
historyId: historyIdDeleted0,
|
||||
fileId: fileIdWithDifferentHashNotFound1,
|
||||
},
|
||||
]
|
||||
const logs = [
|
||||
{
|
||||
projectId: projectId0,
|
||||
fileId: fileIdWithDifferentHashFound,
|
||||
err: { message: 'OError: hash mismatch' },
|
||||
hash: gitBlobHash(fileIdMissing0), // does not matter
|
||||
entry: {
|
||||
ctx: { historyId: historyId0.toString() },
|
||||
hash: gitBlobHash(fileIdInGoodState),
|
||||
},
|
||||
msg: 'failed to process file',
|
||||
},
|
||||
{
|
||||
projectId: projectId0,
|
||||
fileId: fileIdWithDifferentHashRestore,
|
||||
err: { message: 'OError: hash mismatch' },
|
||||
hash: hashDoesNotExistAsBlob,
|
||||
entry: {
|
||||
ctx: { historyId: historyId0.toString() },
|
||||
hash: gitBlobHash(fileIdMissing0), // does not matter
|
||||
},
|
||||
msg: 'failed to process file',
|
||||
},
|
||||
{
|
||||
projectId: projectId0,
|
||||
fileId: fileIdWithDifferentHashNotFound0,
|
||||
err: { message: 'OError: hash mismatch' },
|
||||
hash: gitBlobHash(fileIdWithDifferentHashNotFound0),
|
||||
entry: {
|
||||
ctx: { historyId: historyId0.toString() },
|
||||
hash: hashDoesNotExistAsBlob,
|
||||
},
|
||||
msg: 'failed to process file',
|
||||
},
|
||||
{
|
||||
projectId: projectIdDeleted0,
|
||||
fileId: fileIdWithDifferentHashNotFound1,
|
||||
err: { message: 'OError: hash mismatch' },
|
||||
hash: gitBlobHash(fileIdWithDifferentHashNotFound1),
|
||||
entry: {
|
||||
ctx: { historyId: historyIdDeleted0.toString() },
|
||||
hash: hashDoesNotExistAsBlob,
|
||||
},
|
||||
msg: 'failed to process file',
|
||||
},
|
||||
{
|
||||
projectId: projectId0,
|
||||
fileId: fileIdMissing0,
|
||||
bucketName: USER_FILES_BUCKET_NAME,
|
||||
err: { message: 'NotFoundError' },
|
||||
msg: 'failed to process file',
|
||||
},
|
||||
{
|
||||
projectId: projectId0,
|
||||
fileId: fileIdMissing2,
|
||||
bucketName: USER_FILES_BUCKET_NAME,
|
||||
err: { message: 'NotFoundError' },
|
||||
msg: 'failed to process file',
|
||||
},
|
||||
{
|
||||
projectId: projectIdDeleted0,
|
||||
fileId: fileIdMissing1,
|
||||
bucketName: USER_FILES_BUCKET_NAME,
|
||||
err: { message: 'NotFoundError' },
|
||||
msg: 'failed to process file',
|
||||
},
|
||||
{
|
||||
err: { message: 'spurious error' },
|
||||
msg: 'failed to process file, trying again',
|
||||
},
|
||||
{
|
||||
err: { message: 'some other error' },
|
||||
msg: 'failed to process file',
|
||||
},
|
||||
// from find_malformed_filetrees.mjs
|
||||
{
|
||||
projectId: projectId0,
|
||||
_id: fileIdHashMissing0,
|
||||
reason: 'bad file hash',
|
||||
msg: 'bad file-tree path',
|
||||
},
|
||||
{
|
||||
projectId: projectId0,
|
||||
_id: fileIdHashMissing1,
|
||||
reason: 'bad file hash',
|
||||
msg: 'bad file-tree path',
|
||||
},
|
||||
{
|
||||
projectId: projectId0,
|
||||
_id: fileIdBlobExistsInGCSCorrupted,
|
||||
reason: 'bad file hash',
|
||||
msg: 'bad file-tree path',
|
||||
},
|
||||
]
|
||||
if (PRINT_IDS_AND_HASHES_FOR_DEBUGGING) {
|
||||
const fileIds = {
|
||||
fileIdWithDifferentHashFound,
|
||||
fileIdInGoodState,
|
||||
fileIdWithDifferentHashNotFound0,
|
||||
fileIdWithDifferentHashNotFound1,
|
||||
fileIdMissing0,
|
||||
fileIdMissing1,
|
||||
fileIdMissing2,
|
||||
fileIdWithDifferentHashRestore,
|
||||
fileIdHashMissing0,
|
||||
fileIdHashMissing1,
|
||||
}
|
||||
console.log({
|
||||
projectId0,
|
||||
projectIdDeleted0,
|
||||
historyId0,
|
||||
historyIdDeleted0,
|
||||
...fileIds,
|
||||
hashDoesNotExistAsBlob,
|
||||
})
|
||||
for (const [name, v] of Object.entries(fileIds)) {
|
||||
console.log(
|
||||
name,
|
||||
gitBlobHash(v),
|
||||
Array.from(binaryForGitBlobHash(gitBlobHash(v)).value())
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
before(cleanup.everything)
|
||||
|
||||
before('populate blobs/GCS', async function () {
|
||||
await mockFilestore.start()
|
||||
mockFilestore.addFile(
|
||||
projectId0,
|
||||
fileIdHashMissing0,
|
||||
fileIdHashMissing0.toString()
|
||||
)
|
||||
mockFilestore.addFile(
|
||||
projectId0,
|
||||
fileIdHashMissing1,
|
||||
fileIdHashMissing1.toString()
|
||||
)
|
||||
mockFilestore.addFile(
|
||||
projectId0,
|
||||
fileIdBlobExistsInGCSCorrupted,
|
||||
fileIdBlobExistsInGCSCorrupted.toString()
|
||||
)
|
||||
await new BlobStore(historyId0.toString()).putString(
|
||||
fileIdHashMissing1.toString() // partially processed
|
||||
)
|
||||
const path = '/tmp/test-blob-corrupted'
|
||||
try {
|
||||
await fs.promises.writeFile(path, contentCorruptedBlob)
|
||||
await new BlobStore(historyId0.toString()).putBlob(
|
||||
path,
|
||||
new Blob(gitBlobHash(fileIdBlobExistsInGCSCorrupted), 42)
|
||||
)
|
||||
} finally {
|
||||
await fs.promises.rm(path, { force: true })
|
||||
}
|
||||
await cleanup.postgres()
|
||||
await cleanup.mongo()
|
||||
await Promise.all([
|
||||
testProjects.createEmptyProject(historyId0.toString()),
|
||||
testProjects.createEmptyProject(historyIdDeleted0),
|
||||
])
|
||||
await new BlobStore(historyId0.toString()).putString(
|
||||
fileIdWithDifferentHashNotFound0.toString()
|
||||
)
|
||||
await new BlobStore(historyIdDeleted0.toString()).putString(
|
||||
fileIdWithDifferentHashNotFound1.toString()
|
||||
)
|
||||
await new BlobStore(historyId0.toString()).putString(
|
||||
fileIdInGoodState.toString()
|
||||
)
|
||||
})
|
||||
|
||||
before('populate mongo', async function () {
|
||||
await projectsCollection.insertMany([
|
||||
{
|
||||
_id: projectId0,
|
||||
rootFolder: [
|
||||
{
|
||||
fileRefs: [
|
||||
{ _id: fileIdMissing0 },
|
||||
{ _id: fileIdMissing0 }, // bad file-tree, duplicated fileRef.
|
||||
{ _id: fileIdMissing2 },
|
||||
{ _id: fileIdHashMissing0 },
|
||||
{ _id: fileIdHashMissing1 },
|
||||
{
|
||||
_id: fileIdWithDifferentHashFound,
|
||||
hash: gitBlobHash(fileIdInGoodState),
|
||||
},
|
||||
{
|
||||
_id: fileIdWithDifferentHashRestore,
|
||||
hash: gitBlobHash(fileIdMissing0),
|
||||
},
|
||||
],
|
||||
folders: [
|
||||
{
|
||||
docs: [],
|
||||
},
|
||||
null,
|
||||
{
|
||||
fileRefs: [
|
||||
null,
|
||||
{
|
||||
_id: fileIdInGoodState,
|
||||
hash: gitBlobHash(fileIdInGoodState),
|
||||
},
|
||||
{
|
||||
_id: fileIdWithDifferentHashNotFound0,
|
||||
hash: hashDoesNotExistAsBlob,
|
||||
},
|
||||
{
|
||||
_id: fileIdBlobExistsInGCSCorrupted,
|
||||
hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted),
|
||||
},
|
||||
],
|
||||
folders: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
overleaf: { history: { id: historyId0 } },
|
||||
version: 0,
|
||||
},
|
||||
])
|
||||
await deletedProjectsCollection.insertMany([
|
||||
{
|
||||
_id: deleteProjectsRecordId0,
|
||||
project: {
|
||||
_id: projectIdDeleted0,
|
||||
rootFolder: [
|
||||
{
|
||||
fileRefs: [
|
||||
{
|
||||
_id: fileIdWithDifferentHashNotFound1,
|
||||
hash: hashDoesNotExistAsBlob,
|
||||
},
|
||||
],
|
||||
folders: [
|
||||
{
|
||||
fileRefs: [],
|
||||
folders: [
|
||||
{ fileRefs: [{ _id: fileIdMissing1 }], folders: [] },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
overleaf: { history: { id: historyIdDeleted0 } },
|
||||
version: 100,
|
||||
},
|
||||
deleterData: {
|
||||
deletedProjectId: projectIdDeleted0,
|
||||
},
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
/**
|
||||
* @param {Array<string>} args
|
||||
* @param {Record<string, string>} env
|
||||
* @return {Promise<{ stdout: string, stderr: string, status: number }>}
|
||||
*/
|
||||
async function tryRunScript(args = [], env = {}) {
|
||||
let result
|
||||
try {
|
||||
result = await promisify(execFile)(
|
||||
process.argv0,
|
||||
['storage/scripts/back_fill_file_hash_fix_up.mjs', ...args],
|
||||
{
|
||||
encoding: 'utf-8',
|
||||
timeout: TIMEOUT - 500,
|
||||
env: {
|
||||
...process.env,
|
||||
USER_FILES_BUCKET_NAME,
|
||||
SLEEP_BEFORE_EXIT: '1',
|
||||
...env,
|
||||
LOG_LEVEL: 'warn', // Override LOG_LEVEL of acceptance tests
|
||||
},
|
||||
}
|
||||
)
|
||||
result.status = 0
|
||||
} catch (err) {
|
||||
const { stdout, stderr, code } = err
|
||||
if (typeof code !== 'number') {
|
||||
console.log(err)
|
||||
}
|
||||
result = { stdout, stderr, status: code }
|
||||
}
|
||||
expect((await fs.promises.readdir('/tmp')).join(';')).to.not.match(
|
||||
/back_fill_file_hash/
|
||||
)
|
||||
return result
|
||||
}
|
||||
async function runScriptWithLogs() {
|
||||
const logsPath = '/tmp/test-script-logs'
|
||||
let result
|
||||
try {
|
||||
await fs.promises.writeFile(
|
||||
logsPath,
|
||||
logs.map(e => JSON.stringify(e)).join('\n')
|
||||
)
|
||||
result = await tryRunScript([`--logs=${logsPath}`])
|
||||
} finally {
|
||||
await fs.promises.rm(logsPath, { force: true })
|
||||
}
|
||||
const stats = JSON.parse(result.stdout.trim().split('\n').pop())
|
||||
return {
|
||||
result,
|
||||
stats,
|
||||
}
|
||||
}
|
||||
|
||||
let result, stats
|
||||
before(async function () {
|
||||
;({ result, stats } = await runScriptWithLogs())
|
||||
})
|
||||
it('should print stats', function () {
|
||||
expect(stats).to.contain({
|
||||
processedLines: 12,
|
||||
success: 7,
|
||||
alreadyProcessed: 0,
|
||||
fileDeleted: 0,
|
||||
skipped: 0,
|
||||
failed: 3,
|
||||
unmatched: 1,
|
||||
})
|
||||
})
|
||||
it('should handle re-run on same logs', async function () {
|
||||
;({ stats } = await runScriptWithLogs())
|
||||
expect(stats).to.contain({
|
||||
processedLines: 12,
|
||||
success: 0,
|
||||
alreadyProcessed: 4,
|
||||
fileDeleted: 3,
|
||||
skipped: 0,
|
||||
failed: 3,
|
||||
unmatched: 1,
|
||||
})
|
||||
})
|
||||
it('should flag the unknown fatal error', function () {
|
||||
const unknown = result.stdout
|
||||
.split('\n')
|
||||
.filter(l => l.includes('unknown fatal error'))
|
||||
expect(unknown).to.have.length(1)
|
||||
const [line] = unknown
|
||||
expect(line).to.exist
|
||||
expect(line).to.include('some other error')
|
||||
})
|
||||
it('should flag the unexpected blob on mismatched hash', function () {
|
||||
const line = result.stdout
|
||||
.split('\n')
|
||||
.find(l => l.includes('found blob with computed filestore object hash'))
|
||||
expect(line).to.exist
|
||||
expect(line).to.include(projectId0.toString())
|
||||
expect(line).to.include(fileIdWithDifferentHashFound.toString())
|
||||
expect(line).to.include(gitBlobHash(fileIdInGoodState))
|
||||
})
|
||||
it('should flag the need to restore', function () {
|
||||
const line = result.stdout
|
||||
.split('\n')
|
||||
.find(l => l.includes('missing blob, need to restore filestore file'))
|
||||
expect(line).to.exist
|
||||
expect(line).to.include(projectId0.toString())
|
||||
expect(line).to.include(fileIdWithDifferentHashRestore.toString())
|
||||
expect(line).to.include(hashDoesNotExistAsBlob)
|
||||
})
|
||||
it('should flag the corrupted blob', function () {
|
||||
const line = result.stdout
|
||||
.split('\n')
|
||||
.find(l => l.includes('blob corrupted'))
|
||||
expect(line).to.exist
|
||||
expect(line).to.include(projectId0.toString())
|
||||
expect(line).to.include(fileIdBlobExistsInGCSCorrupted.toString())
|
||||
expect(line).to.include(
|
||||
gitBlobHashBuffer(Buffer.from(contentCorruptedBlob))
|
||||
)
|
||||
expect(line).to.include(gitBlobHash(fileIdBlobExistsInGCSCorrupted))
|
||||
})
|
||||
it('should update mongo', async function () {
|
||||
expect(await projectsCollection.find({}).toArray())
|
||||
.excludingEvery([
|
||||
'currentEndTimestamp',
|
||||
'currentEndVersion',
|
||||
'updatedAt',
|
||||
'backup',
|
||||
])
|
||||
.to.deep.equal([
|
||||
{
|
||||
_id: projectId0,
|
||||
rootFolder: [
|
||||
{
|
||||
fileRefs: [
|
||||
// Removed
|
||||
// { _id: fileIdMissing0 },
|
||||
// Removed
|
||||
// { _id: fileIdMissing2 },
|
||||
// Added hash
|
||||
{
|
||||
_id: fileIdHashMissing0,
|
||||
hash: gitBlobHash(fileIdHashMissing0),
|
||||
},
|
||||
// Added hash
|
||||
{
|
||||
_id: fileIdHashMissing1,
|
||||
hash: gitBlobHash(fileIdHashMissing1),
|
||||
},
|
||||
// No change, should warn about the find.
|
||||
{
|
||||
_id: fileIdWithDifferentHashFound,
|
||||
hash: gitBlobHash(fileIdInGoodState),
|
||||
},
|
||||
// No change, should warn about the need to restore.
|
||||
{
|
||||
_id: fileIdWithDifferentHashRestore,
|
||||
hash: gitBlobHash(fileIdMissing0),
|
||||
},
|
||||
],
|
||||
folders: [
|
||||
{
|
||||
docs: [],
|
||||
},
|
||||
null,
|
||||
{
|
||||
fileRefs: [
|
||||
null,
|
||||
// No change
|
||||
{
|
||||
_id: fileIdInGoodState,
|
||||
hash: gitBlobHash(fileIdInGoodState),
|
||||
},
|
||||
// Updated hash
|
||||
{
|
||||
_id: fileIdWithDifferentHashNotFound0,
|
||||
hash: gitBlobHash(fileIdWithDifferentHashNotFound0),
|
||||
},
|
||||
// No change, flagged
|
||||
{
|
||||
_id: fileIdBlobExistsInGCSCorrupted,
|
||||
hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted),
|
||||
},
|
||||
],
|
||||
folders: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
overleaf: { history: { id: historyId0 } },
|
||||
// Incremented when removing file/updating hash
|
||||
version: 5,
|
||||
},
|
||||
])
|
||||
expect(await deletedProjectsCollection.find({}).toArray()).to.deep.equal([
|
||||
{
|
||||
_id: deleteProjectsRecordId0,
|
||||
project: {
|
||||
_id: projectIdDeleted0,
|
||||
rootFolder: [
|
||||
{
|
||||
fileRefs: [
|
||||
// Updated hash
|
||||
{
|
||||
_id: fileIdWithDifferentHashNotFound1,
|
||||
hash: gitBlobHash(fileIdWithDifferentHashNotFound1),
|
||||
},
|
||||
],
|
||||
folders: [
|
||||
{
|
||||
fileRefs: [],
|
||||
folders: [
|
||||
{
|
||||
fileRefs: [
|
||||
// Removed
|
||||
// { _id: fileIdMissing1 },
|
||||
],
|
||||
folders: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
overleaf: { history: { id: historyIdDeleted0 } },
|
||||
// Incremented when removing file/updating hash
|
||||
version: 102,
|
||||
},
|
||||
deleterData: {
|
||||
deletedProjectId: projectIdDeleted0,
|
||||
},
|
||||
},
|
||||
])
|
||||
const writtenBlobsByProject = new Map()
|
||||
for (const { projectId, fileId } of writtenBlobs) {
|
||||
writtenBlobsByProject.set(
|
||||
projectId,
|
||||
(writtenBlobsByProject.get(projectId) || []).concat([fileId])
|
||||
)
|
||||
}
|
||||
})
|
||||
it('should have written the back filled files to history v1', async function () {
|
||||
for (const { historyId, fileId } of writtenBlobs) {
|
||||
const blobStore = new BlobStore(historyId.toString())
|
||||
const hash = gitBlobHash(fileId.toString())
|
||||
const blob = await blobStore.getBlob(hash)
|
||||
expect(blob).to.exist
|
||||
expect(blob.getByteLength()).to.equal(24)
|
||||
const id = await blobStore.getString(hash)
|
||||
expect(id).to.equal(fileId.toString())
|
||||
// double check we are not comparing 'undefined' or '[object Object]' above
|
||||
expect(id).to.match(/^[a-f0-9]{24}$/)
|
||||
}
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user