From 6ed488cc650b1c312872c00a71b2e8ac92163aa6 Mon Sep 17 00:00:00 2001 From: Andrew Rumble Date: Fri, 6 Jun 2025 12:26:16 +0100 Subject: [PATCH] Change raw mode to just fetch all raw content from backup GitOrigin-RevId: 397060d63c824dbe688d187312dd78d1f3e4bf5d --- .../history-v1/storage/lib/backupArchiver.mjs | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/services/history-v1/storage/lib/backupArchiver.mjs b/services/history-v1/storage/lib/backupArchiver.mjs index 30a9708879..4e762df1ba 100644 --- a/services/history-v1/storage/lib/backupArchiver.mjs +++ b/services/history-v1/storage/lib/backupArchiver.mjs @@ -374,6 +374,41 @@ export async function archiveLatestChunk( return archive } +/** + * Fetches all raw blobs from the project and adds them to the archive. + * + * @param {string} historyId + * @param {Archiver} archive + * @param {CachedPerProjectEncryptedS3Persistor} projectCache + * @return {Promise} + */ +async function addRawBlobsToArchive(historyId, archive, projectCache) { + const key = projectKey.format(historyId) + const { contents } = await projectCache.listDirectory(projectBlobsBucket, key) + for (const blobRecord of contents) { + if (!blobRecord.Key) { + logger.debug({ blobRecord }, 'no key') + continue + } + const blobKey = blobRecord.Key + try { + const stream = await projectCache.getObjectStream( + projectBlobsBucket, + blobKey, + { autoGunzip: true } + ) + archive.append(stream, { + name: path.join(historyId, 'blobs', blobKey), + }) + } catch (err) { + logger.warn( + { err, path: blobRecord.Key }, + 'Failed to append blob to archive' + ) + } + } +} + /** * Download raw files from the backup. * @@ -410,22 +445,13 @@ export async function archiveRawProject( const chunkId = chunkRecord.Key.split('/').pop() logger.debug({ chunkId, key: chunkRecord.Key }, 'Processing chunk') - const { chunkData, buffer } = await loadChunkByKey( - projectCache, - chunkRecord.Key - ) + const { buffer } = await loadChunkByKey(projectCache, chunkRecord.Key) archive.append(buffer, { name: `${historyId}/chunks/${chunkId}/chunk.json`, }) - - const chunk = History.fromRaw(chunkData) - - await addChunkToArchive(chunk, archive, projectCache, historyId, { - prefix: `${historyId}/chunks/${chunkId}/`, - useBackupGlobalBlobs, - }) } + await addRawBlobsToArchive(historyId, archive, projectCache) } export class BackupPersistorError extends OError {}