mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-14 06:03:45 +01:00
Merge pull request #29952 from overleaf/bg-history-extend-backup-comparison-II
Add batch comparison options for backup verification GitOrigin-RevId: fe12270b4063c4105f7e7a950c6a58cd39a62843
This commit is contained in:
Brian Gough
@@ -376,6 +376,19 @@ const optionDefinitions = [
|
||||
description:
|
||||
'Performs a fast comparison of blobs by only checking for presence and size. Only works with --compare.',
|
||||
},
|
||||
{
|
||||
name: 'input',
|
||||
type: String,
|
||||
description:
|
||||
'Input file containing project IDs (one per line) for batch comparison. Only works with --compare.',
|
||||
},
|
||||
{
|
||||
name: 'verbose',
|
||||
alias: 'v',
|
||||
type: Boolean,
|
||||
description:
|
||||
'Enable verbose output during batch comparison. Only works with --compare and --input.',
|
||||
},
|
||||
]
|
||||
|
||||
function handleOptions() {
|
||||
@@ -394,7 +407,8 @@ function handleOptions() {
|
||||
!options.pending &&
|
||||
!options.init &&
|
||||
!(options.fix >= 0) &&
|
||||
!(options.compare && options['start-date'] && options['end-date'])
|
||||
!(options.compare && options['start-date'] && options['end-date']) &&
|
||||
!(options.compare && options.input)
|
||||
|
||||
if (projectIdRequired && !options.projectId) {
|
||||
console.error('Error: projectId is required')
|
||||
@@ -434,10 +448,11 @@ function handleOptions() {
|
||||
if (
|
||||
options.compare &&
|
||||
!options.projectId &&
|
||||
!(options['start-date'] && options['end-date'])
|
||||
!(options['start-date'] && options['end-date']) &&
|
||||
!options.input
|
||||
) {
|
||||
console.error(
|
||||
'Error: --compare requires either projectId or both --start-date and --end-date'
|
||||
'Error: --compare requires either projectId, --input file, or both --start-date and --end-date'
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
@@ -447,6 +462,28 @@ function handleOptions() {
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (options.input && !options.compare) {
|
||||
console.error('Error: --input can only be used with --compare')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (options.input && options.projectId) {
|
||||
console.error('Error: --input cannot be specified with projectId')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (options.input && (options['start-date'] || options['end-date'])) {
|
||||
console.error(
|
||||
'Error: --input cannot be specified with --start-date or --end-date'
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (options.verbose && !options.input) {
|
||||
console.error('Error: --verbose can only be used with --input')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
DRY_RUN = options['dry-run'] || false
|
||||
RETRY_LIMIT = options.retries || 3
|
||||
CONCURRENCY = options.concurrency || 1
|
||||
@@ -898,8 +935,21 @@ async function getBlobListing(historyId) {
|
||||
return remoteBlobs
|
||||
}
|
||||
|
||||
async function compareBackups(projectId, options) {
|
||||
console.log(`Comparing backups for project ${projectId}`)
|
||||
/**
|
||||
* @typedef {Object} ComparisonError
|
||||
* @property {string} type - Error type code (e.g., 'chunk-not-found', 'blob-hash-mismatch')
|
||||
* @property {string} [chunkId]
|
||||
* @property {string} historyId
|
||||
* @property {string} [blobHash]
|
||||
* @property {string|Error} error
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Error & {historyId: string, errors: ComparisonError[], counters: Object}} ComparisonFailureError
|
||||
*/
|
||||
|
||||
async function compareBackups(projectId, options, log = console.log) {
|
||||
log(`Comparing backups for project ${projectId}`)
|
||||
const { historyId } = await getBackupStatus(projectId)
|
||||
const chunks = await getProjectChunks(historyId)
|
||||
const blobStore = new BlobStore(historyId)
|
||||
@@ -914,6 +964,7 @@ async function compareBackups(projectId, options) {
|
||||
let totalBlobMatches = 0
|
||||
let totalBlobMismatches = 0
|
||||
let totalBlobsNotFound = 0
|
||||
/** @type {ComparisonError[]} */
|
||||
const errors = []
|
||||
const blobComparator = new BlobComparator(backupPersistorForProject)
|
||||
|
||||
@@ -938,33 +989,48 @@ async function compareBackups(projectId, options) {
|
||||
const backupEndVersion = chunk.startVersion + backupChunk.changes.length
|
||||
|
||||
if (originalStr === backupStr) {
|
||||
console.log(
|
||||
log(
|
||||
`✓ Chunk ${chunk.id} (v${chunk.startVersion}-v${chunk.endVersion}) matches`
|
||||
)
|
||||
totalChunkMatches++
|
||||
} else if (originalStr === JSON.stringify(JSON.parse(backupStr))) {
|
||||
console.log(
|
||||
log(
|
||||
`✓ Chunk ${chunk.id} (v${chunk.startVersion}-v${chunk.endVersion}) matches (after normalisation)`
|
||||
)
|
||||
totalChunkMatches++
|
||||
} else if (backupEndVersion < chunk.endVersion) {
|
||||
console.log(
|
||||
log(
|
||||
`✗ Chunk ${chunk.id} is ahead of backup (v${chunk.startVersion}-v${chunk.endVersion} vs v${backupStartVersion}-v${backupEndVersion})`
|
||||
)
|
||||
totalChunkMismatches++
|
||||
errors.push({ chunkId: chunk.id, error: 'Chunk ahead of backup' })
|
||||
errors.push({
|
||||
type: 'chunk-ahead',
|
||||
chunkId: chunk.id,
|
||||
historyId,
|
||||
error: 'Chunk ahead of backup',
|
||||
})
|
||||
} else {
|
||||
console.log(
|
||||
log(
|
||||
`✗ Chunk ${chunk.id} (v${chunk.startVersion}-v${chunk.endVersion}) MISMATCH`
|
||||
)
|
||||
totalChunkMismatches++
|
||||
errors.push({ chunkId: chunk.id, error: 'Chunk mismatch' })
|
||||
errors.push({
|
||||
type: 'chunk-mismatch',
|
||||
chunkId: chunk.id,
|
||||
historyId,
|
||||
error: 'Chunk mismatch',
|
||||
})
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof NotFoundError) {
|
||||
console.log(`✗ Chunk ${chunk.id} not found in backup`, err.cause)
|
||||
log(`✗ Chunk ${chunk.id} not found in backup`, err.cause)
|
||||
totalChunksNotFound++
|
||||
errors.push({ chunkId: chunk.id, error: `Chunk not found` })
|
||||
errors.push({
|
||||
type: 'chunk-not-found',
|
||||
chunkId: chunk.id,
|
||||
historyId,
|
||||
error: `Chunk not found`,
|
||||
})
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
@@ -983,7 +1049,7 @@ async function compareBackups(projectId, options) {
|
||||
|
||||
if (GLOBAL_BLOBS.has(blob.hash)) {
|
||||
const globalBlob = GLOBAL_BLOBS.get(blob.hash)
|
||||
console.log(
|
||||
log(
|
||||
` ✓ Blob ${blob.hash} is a global blob`,
|
||||
globalBlob?.demoted ? '(demoted)' : ''
|
||||
)
|
||||
@@ -995,7 +1061,7 @@ async function compareBackups(projectId, options) {
|
||||
if (blobListEntry) {
|
||||
if (blob.byteLength === blobListEntry.Size) {
|
||||
// Size matches exactly
|
||||
console.log(
|
||||
log(
|
||||
` ✓ Blob ${blob.hash} exists on remote with expected size (${blob.byteLength} bytes)`
|
||||
)
|
||||
totalBlobMatches++
|
||||
@@ -1005,29 +1071,35 @@ async function compareBackups(projectId, options) {
|
||||
const compressionRatio = (
|
||||
blobListEntry.Size / blob.byteLength
|
||||
).toFixed(2)
|
||||
console.log(
|
||||
log(
|
||||
` ✓ Blob ${blob.hash} consistent with compressed data on remote (${blob.byteLength} bytes => ${blobListEntry.Size} bytes, ratio=${compressionRatio})`
|
||||
)
|
||||
totalBlobMatches++
|
||||
continue
|
||||
} else {
|
||||
console.log(
|
||||
log(
|
||||
` ✗ Blob ${blob.hash} size mismatch (original: ${blob.byteLength} bytes, stringLength: ${blob.stringLength}, backup: ${blobListEntry.Size} bytes)`
|
||||
)
|
||||
totalBlobMismatches++
|
||||
errors.push({
|
||||
type: 'blob-size-mismatch',
|
||||
chunkId: chunk.id,
|
||||
historyId,
|
||||
blobHash: blob.hash,
|
||||
error: `Blob ${blob.hash} size mismatch`,
|
||||
})
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
log(
|
||||
` ✗ Blob ${blob.hash} not found on remote listing (${blob.byteLength} bytes, ${blob.stringLength} string length)`
|
||||
)
|
||||
totalBlobMismatches++
|
||||
errors.push({
|
||||
type: 'blob-not-found',
|
||||
chunkId: chunk.id,
|
||||
historyId,
|
||||
blobHash: blob.hash,
|
||||
error: `Blob ${blob.hash} not found`,
|
||||
})
|
||||
continue
|
||||
@@ -1037,20 +1109,23 @@ async function compareBackups(projectId, options) {
|
||||
await blobComparator.compareBlob(historyId, blob)
|
||||
|
||||
if (matches) {
|
||||
console.log(
|
||||
log(
|
||||
` ✓ Blob ${blob.hash} hash matches (${blob.byteLength} bytes)` +
|
||||
(fromCache ? ' (from cache)' : '')
|
||||
)
|
||||
totalBlobMatches++
|
||||
continue
|
||||
} else {
|
||||
console.log(
|
||||
log(
|
||||
` ✗ Blob ${blob.hash} hash mismatch (original: ${blob.hash}, backup: ${computedHash}) (${blob.byteLength} bytes, ${blob.stringLength} string length)` +
|
||||
(fromCache ? ' (from cache)' : '')
|
||||
)
|
||||
totalBlobMismatches++
|
||||
errors.push({
|
||||
type: 'blob-hash-mismatch',
|
||||
chunkId: chunk.id,
|
||||
historyId,
|
||||
blobHash: blob.hash,
|
||||
error: `Blob ${blob.hash} hash mismatch`,
|
||||
})
|
||||
continue
|
||||
@@ -1058,10 +1133,13 @@ async function compareBackups(projectId, options) {
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof NotFoundError) {
|
||||
console.log(` ✗ Blob ${blob.hash} not found in backup`, err.cause)
|
||||
log(` ✗ Blob ${blob.hash} not found in backup`, err.cause)
|
||||
totalBlobsNotFound++
|
||||
errors.push({
|
||||
type: 'blob-not-found',
|
||||
chunkId: chunk.id,
|
||||
historyId,
|
||||
blobHash: blob.hash,
|
||||
error: `Blob ${blob.hash} not found`,
|
||||
})
|
||||
} else {
|
||||
@@ -1070,31 +1148,197 @@ async function compareBackups(projectId, options) {
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error comparing chunk ${chunk.id}:`, err)
|
||||
errors.push({ chunkId: chunk.id, error: err })
|
||||
log(`Error comparing chunk ${chunk.id}:`, err)
|
||||
errors.push({
|
||||
type: 'error',
|
||||
chunkId: chunk.id,
|
||||
historyId,
|
||||
error: err instanceof Error ? err : String(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Print summary
|
||||
console.log('\nComparison Summary:')
|
||||
console.log('==================')
|
||||
console.log(`Total chunks: ${chunks.length}`)
|
||||
console.log(`Chunk matches: ${totalChunkMatches}`)
|
||||
console.log(`Chunk mismatches: ${totalChunkMismatches}`)
|
||||
console.log(`Chunk not found: ${totalChunksNotFound}`)
|
||||
console.log(`Blob matches: ${totalBlobMatches}`)
|
||||
console.log(`Blob mismatches: ${totalBlobMismatches}`)
|
||||
console.log(`Blob not found: ${totalBlobsNotFound}`)
|
||||
console.log(`Errors: ${errors.length}`)
|
||||
log('\nComparison Summary:')
|
||||
log('==================')
|
||||
log(`Total chunks: ${chunks.length}`)
|
||||
log(`Chunk matches: ${totalChunkMatches}`)
|
||||
log(`Chunk mismatches: ${totalChunkMismatches}`)
|
||||
log(`Chunk not found: ${totalChunksNotFound}`)
|
||||
log(`Blob matches: ${totalBlobMatches}`)
|
||||
log(`Blob mismatches: ${totalBlobMismatches}`)
|
||||
log(`Blob not found: ${totalBlobsNotFound}`)
|
||||
log(`Errors: ${errors.length}`)
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.log('\nErrors:')
|
||||
log('\nErrors:')
|
||||
errors.forEach(({ chunkId, error }) => {
|
||||
console.log(` Chunk ${chunkId}: ${error}`)
|
||||
log(` Chunk ${chunkId}: ${error}`)
|
||||
})
|
||||
throw new Error('Backup comparison FAILED')
|
||||
const err = /** @type {ComparisonFailureError} */ (
|
||||
new Error('Backup comparison FAILED')
|
||||
)
|
||||
err.historyId = historyId
|
||||
err.errors = errors
|
||||
err.counters = {
|
||||
totalChunks: chunks.length,
|
||||
chunkMatches: totalChunkMatches,
|
||||
chunkMismatches: totalChunkMismatches,
|
||||
chunksNotFound: totalChunksNotFound,
|
||||
blobMatches: totalBlobMatches,
|
||||
blobMismatches: totalBlobMismatches,
|
||||
blobsNotFound: totalBlobsNotFound,
|
||||
}
|
||||
throw err
|
||||
} else {
|
||||
console.log('Backup comparison successful')
|
||||
log('Backup comparison successful')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare a single project and emit structured output
|
||||
* @param {string} projectId - The project ID to compare
|
||||
* @param {Object} options - Comparison options
|
||||
* @param {number} projectNumber - Current project number for progress reporting
|
||||
* @param {number} totalCount - Total number of projects
|
||||
* @returns {Promise<boolean>} - Returns true if comparison had errors
|
||||
*/
|
||||
async function compareProjectAndEmitResult(
|
||||
projectId,
|
||||
options,
|
||||
projectNumber,
|
||||
totalCount
|
||||
) {
|
||||
if (gracefulShutdownInitiated) {
|
||||
return false
|
||||
}
|
||||
|
||||
console.error(
|
||||
`Processing project ${projectNumber}/${totalCount}: ${projectId}`
|
||||
)
|
||||
|
||||
// Custom logger: silent by default, buffered if verbose
|
||||
const logBuffer = []
|
||||
const customLog = options.verbose
|
||||
? (...args) => logBuffer.push(args.join(' '))
|
||||
: () => {}
|
||||
|
||||
try {
|
||||
await compareBackups(projectId, options, customLog)
|
||||
console.log(`OK: ${projectId}`)
|
||||
|
||||
// Output buffered logs after success
|
||||
if (options.verbose && logBuffer.length > 0) {
|
||||
console.error(`\n--- Verbose output for ${projectId} ---`)
|
||||
logBuffer.forEach(line => console.error(line))
|
||||
console.error(`--- End of output for ${projectId} ---\n`)
|
||||
}
|
||||
|
||||
return false
|
||||
} catch (err) {
|
||||
console.log(`FAIL: ${projectId}`)
|
||||
|
||||
// Output buffered logs on error when verbose
|
||||
if (options.verbose && logBuffer.length > 0) {
|
||||
console.error(`\n--- Verbose output for ${projectId} (FAILED) ---`)
|
||||
logBuffer.forEach(line => console.error(line))
|
||||
console.error(`--- End of output for ${projectId} ---\n`)
|
||||
}
|
||||
|
||||
// Check if this is a comparison error with attached details
|
||||
const error = /** @type {ComparisonFailureError} */ (err)
|
||||
if (error.errors && error.historyId) {
|
||||
// Emit structured error lines
|
||||
for (const errorRecord of error.errors) {
|
||||
const {
|
||||
type,
|
||||
historyId,
|
||||
blobHash,
|
||||
chunkId,
|
||||
error: errorDetail,
|
||||
} = errorRecord
|
||||
const errorMsg =
|
||||
typeof errorDetail === 'string'
|
||||
? errorDetail
|
||||
: errorDetail?.message || String(errorDetail)
|
||||
|
||||
// Use error type for structured output
|
||||
switch (type) {
|
||||
case 'blob-not-found':
|
||||
console.log(`missing: ${projectId},${historyId},${blobHash}`)
|
||||
break
|
||||
case 'chunk-not-found':
|
||||
console.log(`chunk-missing: ${projectId},${historyId},${chunkId}`)
|
||||
break
|
||||
case 'blob-hash-mismatch':
|
||||
console.log(`hash-mismatch: ${projectId},${historyId},${blobHash}`)
|
||||
break
|
||||
case 'blob-size-mismatch':
|
||||
console.log(`size-mismatch: ${projectId},${historyId},${blobHash}`)
|
||||
break
|
||||
case 'chunk-mismatch':
|
||||
console.log(`chunk-mismatch: ${projectId},${historyId},${chunkId}`)
|
||||
break
|
||||
case 'chunk-ahead':
|
||||
console.log(`chunk-ahead: ${projectId},${historyId},${chunkId}`)
|
||||
break
|
||||
default:
|
||||
console.log(
|
||||
`error: ${projectId},${historyId},${errorMsg.replace(/[,\n]/g, ' ')}`
|
||||
)
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Generic error without details
|
||||
const errorMsg = error?.message || String(error)
|
||||
console.log(
|
||||
`error: ${projectId},unknown,${errorMsg.replace(/[,\n]/g, ' ')}`
|
||||
)
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
async function compareProjectsFromFile(options) {
|
||||
await ensureGlobalBlobsLoaded()
|
||||
const limiter = pLimit(CONCURRENCY)
|
||||
let totalErrors = 0
|
||||
let totalProjects = 0
|
||||
|
||||
// Read project IDs from file
|
||||
const fileContent = await fs.readFile(options.input, 'utf-8')
|
||||
const projectIds = fileContent
|
||||
.split('\n')
|
||||
.map(line => line.trim())
|
||||
.filter(line => line.length > 0)
|
||||
|
||||
console.error(`Loaded ${projectIds.length} project IDs from ${options.input}`)
|
||||
|
||||
const operations = projectIds.map(projectId =>
|
||||
limiter(async () => {
|
||||
totalProjects++
|
||||
const hadError = await compareProjectAndEmitResult(
|
||||
projectId,
|
||||
options,
|
||||
totalProjects,
|
||||
projectIds.length
|
||||
)
|
||||
if (hadError) {
|
||||
totalErrors++
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
await Promise.allSettled(operations)
|
||||
|
||||
console.error('\nComparison Summary:')
|
||||
console.error('==================')
|
||||
console.error(`Total projects processed: ${totalProjects}`)
|
||||
console.error(`Projects with errors: ${totalErrors}`)
|
||||
|
||||
if (totalErrors > 0) {
|
||||
throw new Error('Some project comparisons failed')
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1169,7 +1413,9 @@ async function main() {
|
||||
} else if (options.init) {
|
||||
await initializeProjects(options)
|
||||
} else if (options.compare) {
|
||||
if (options['start-date'] && options['end-date']) {
|
||||
if (options.input) {
|
||||
await compareProjectsFromFile(options)
|
||||
} else if (options['start-date'] && options['end-date']) {
|
||||
await compareAllProjects(options)
|
||||
} else {
|
||||
await compareBackups(projectId, options)
|
||||
|
||||
Reference in New Issue
Block a user