Change raw mode to just fetch all raw content from backup

GitOrigin-RevId: 397060d63c824dbe688d187312dd78d1f3e4bf5d
This commit is contained in:
Andrew Rumble
2025-06-06 12:26:16 +01:00
committed by Copybot
parent f2b0a982ac
commit 6ed488cc65

View File

@@ -374,6 +374,41 @@ export async function archiveLatestChunk(
return archive
}
/**
* Fetches all raw blobs from the project and adds them to the archive.
*
* @param {string} historyId
* @param {Archiver} archive
* @param {CachedPerProjectEncryptedS3Persistor} projectCache
* @return {Promise<void>}
*/
async function addRawBlobsToArchive(historyId, archive, projectCache) {
const key = projectKey.format(historyId)
const { contents } = await projectCache.listDirectory(projectBlobsBucket, key)
for (const blobRecord of contents) {
if (!blobRecord.Key) {
logger.debug({ blobRecord }, 'no key')
continue
}
const blobKey = blobRecord.Key
try {
const stream = await projectCache.getObjectStream(
projectBlobsBucket,
blobKey,
{ autoGunzip: true }
)
archive.append(stream, {
name: path.join(historyId, 'blobs', blobKey),
})
} catch (err) {
logger.warn(
{ err, path: blobRecord.Key },
'Failed to append blob to archive'
)
}
}
}
/**
* Download raw files from the backup.
*
@@ -410,22 +445,13 @@ export async function archiveRawProject(
const chunkId = chunkRecord.Key.split('/').pop()
logger.debug({ chunkId, key: chunkRecord.Key }, 'Processing chunk')
const { chunkData, buffer } = await loadChunkByKey(
projectCache,
chunkRecord.Key
)
const { buffer } = await loadChunkByKey(projectCache, chunkRecord.Key)
archive.append(buffer, {
name: `${historyId}/chunks/${chunkId}/chunk.json`,
})
const chunk = History.fromRaw(chunkData)
await addChunkToArchive(chunk, archive, projectCache, historyId, {
prefix: `${historyId}/chunks/${chunkId}/`,
useBackupGlobalBlobs,
})
}
await addRawBlobsToArchive(historyId, archive, projectCache)
}
export class BackupPersistorError extends OError {}