From a9cb0cc197882763f106e46e55deb440e76cd178 Mon Sep 17 00:00:00 2001 From: Eric Mc Sween <5454374+emcsween@users.noreply.github.com> Date: Thu, 8 May 2025 11:27:39 -0400 Subject: [PATCH] Merge pull request #25361 from overleaf/em-load-latest-raw Rename loadLatestRaw() to getLatestChunkMetadata() GitOrigin-RevId: e089dcfa57cbbc43df8666b51eca0d81d595a5a7 --- .../history-v1/api/controllers/projects.js | 2 +- .../storage/lib/chunk_store/index.js | 20 +++++++++---------- .../history-v1/storage/scripts/backup.mjs | 6 +++--- .../acceptance/js/api/backupVerifier.test.mjs | 6 +++--- .../acceptance/js/storage/backup.test.mjs | 12 +++++------ .../acceptance/js/storage/chunk_store.test.js | 14 +++++++------ 6 files changed, 31 insertions(+), 29 deletions(-) diff --git a/services/history-v1/api/controllers/projects.js b/services/history-v1/api/controllers/projects.js index d42e29ab81..5ae74cb6da 100644 --- a/services/history-v1/api/controllers/projects.js +++ b/services/history-v1/api/controllers/projects.js @@ -91,7 +91,7 @@ async function getLatestHistoryRaw(req, res, next) { const readOnly = req.swagger.params.readOnly.value try { const { startVersion, endVersion, endTimestamp } = - await chunkStore.loadLatestRaw(projectId, { readOnly }) + await chunkStore.getLatestChunkMetadata(projectId, { readOnly }) res.json({ startVersion, endVersion, diff --git a/services/history-v1/storage/lib/chunk_store/index.js b/services/history-v1/storage/lib/chunk_store/index.js index 89efaf87ca..12c0924707 100644 --- a/services/history-v1/storage/lib/chunk_store/index.js +++ b/services/history-v1/storage/lib/chunk_store/index.js @@ -96,15 +96,15 @@ async function lazyLoadHistoryFiles(history, batchBlobStore) { * @param {boolean} [opts.readOnly] * @return {Promise<{id: string, startVersion: number, endVersion: number, endTimestamp: Date}>} */ -async function loadLatestRaw(projectId, opts) { +async function getLatestChunkMetadata(projectId, opts) { assert.projectId(projectId, 'bad projectId') const backend = getBackend(projectId) - const chunkRecord = await backend.getLatestChunk(projectId, opts) - if (chunkRecord == null) { + const chunkMetadata = await backend.getLatestChunk(projectId, opts) + if (chunkMetadata == null) { throw new Chunk.NotFoundError(projectId) } - return chunkRecord + return chunkMetadata } /** @@ -116,14 +116,14 @@ async function loadLatestRaw(projectId, opts) { * @return {Promise} */ async function loadLatest(projectId, opts = {}) { - const chunkRecord = await loadLatestRaw(projectId) - const rawHistory = await historyStore.loadRaw(projectId, chunkRecord.id) + const chunkMetadata = await getLatestChunkMetadata(projectId) + const rawHistory = await historyStore.loadRaw(projectId, chunkMetadata.id) const history = History.fromRaw(rawHistory) if (!opts.persistedOnly) { const nonPersistedChanges = await getChunkExtension( projectId, - chunkRecord.endVersion + chunkMetadata.endVersion ) history.pushChanges(nonPersistedChanges) } @@ -131,7 +131,7 @@ async function loadLatest(projectId, opts = {}) { const blobStore = new BlobStore(projectId) const batchBlobStore = new BatchBlobStore(blobStore) await lazyLoadHistoryFiles(history, batchBlobStore) - return new Chunk(history, chunkRecord.startVersion) + return new Chunk(history, chunkMetadata.startVersion) } /** @@ -354,7 +354,7 @@ async function loadByChunkRecord(projectId, chunkRecord) { */ async function* getProjectChunksFromVersion(projectId, version) { const backend = getBackend(projectId) - const latestChunkMetadata = await loadLatestRaw(projectId) + const latestChunkMetadata = await getLatestChunkMetadata(projectId) if (!latestChunkMetadata || version > latestChunkMetadata.endVersion) { return } @@ -500,7 +500,7 @@ module.exports = { getBackend, initializeProject, loadLatest, - loadLatestRaw, + getLatestChunkMetadata, loadAtVersion, loadAtTimestamp, loadByChunkRecord, diff --git a/services/history-v1/storage/scripts/backup.mjs b/services/history-v1/storage/scripts/backup.mjs index 9ae6101105..94df24f66d 100644 --- a/services/history-v1/storage/scripts/backup.mjs +++ b/services/history-v1/storage/scripts/backup.mjs @@ -5,7 +5,7 @@ import commandLineArgs from 'command-line-args' import { Chunk, History, Snapshot } from 'overleaf-editor-core' import { getProjectChunks, - loadLatestRaw, + getLatestChunkMetadata, create, } from '../lib/chunk_store/index.js' import { client } from '../lib/mongodb.js' @@ -444,7 +444,7 @@ async function analyseBackupStatus(projectId) { await getBackupStatus(projectId) // TODO: when we have confidence that the latestChunkMetadata always matches // the values from the backupStatus we can skip loading it here - const latestChunkMetadata = await loadLatestRaw(historyId, { + const latestChunkMetadata = await getLatestChunkMetadata(historyId, { readOnly: Boolean(USE_SECONDARY), }) if ( @@ -454,7 +454,7 @@ async function analyseBackupStatus(projectId) { // compare the current end version with the latest chunk metadata to check that // the updates to the project collection are reliable // expect some failures due to the time window between getBackupStatus and - // loadLatestRaw where the project is being actively edited. + // getLatestChunkMetadata where the project is being actively edited. logger.warn( { projectId, diff --git a/services/history-v1/test/acceptance/js/api/backupVerifier.test.mjs b/services/history-v1/test/acceptance/js/api/backupVerifier.test.mjs index 8c351a2652..44e1a2652b 100644 --- a/services/history-v1/test/acceptance/js/api/backupVerifier.test.mjs +++ b/services/history-v1/test/acceptance/js/api/backupVerifier.test.mjs @@ -119,17 +119,17 @@ async function verifyBlobHTTP(historyId, hash) { } async function backupChunk(historyId) { - const newChunk = await chunkStore.loadLatestRaw(historyId) + const newChunkMetadata = await chunkStore.getLatestChunkMetadata(historyId) const { buffer: chunkBuffer } = await historyStore.loadRawWithBuffer( historyId, - newChunk.id + newChunkMetadata.id ) const md5 = Crypto.createHash('md5').update(chunkBuffer) await backupPersistor.sendStream( chunksBucket, path.join( projectKey.format(historyId), - projectKey.pad(newChunk.startVersion) + projectKey.pad(newChunkMetadata.startVersion) ), Stream.Readable.from([chunkBuffer]), { diff --git a/services/history-v1/test/acceptance/js/storage/backup.test.mjs b/services/history-v1/test/acceptance/js/storage/backup.test.mjs index 83087a1384..fdca1ce294 100644 --- a/services/history-v1/test/acceptance/js/storage/backup.test.mjs +++ b/services/history-v1/test/acceptance/js/storage/backup.test.mjs @@ -169,8 +169,8 @@ describe('backup script', function () { makeChunkKey(historyId, 0) ) const chunkContent = await text(chunkStream.pipe(createGunzip())) - const chunk = await ChunkStore.loadLatestRaw(historyId) - const rawHistory = await historyStore.loadRaw(historyId, chunk.id) + const chunkMetadata = await ChunkStore.getLatestChunkMetadata(historyId) + const rawHistory = await historyStore.loadRaw(historyId, chunkMetadata.id) expect(JSON.parse(chunkContent)).to.deep.equal(rawHistory) // Unrelated entries from backedUpBlobs should be not cleared @@ -299,8 +299,8 @@ describe('backup script', function () { makeChunkKey(historyId, 0) ) const chunkContent = await text(chunkStream.pipe(createGunzip())) - const chunk = await ChunkStore.loadLatestRaw(historyId) - const rawHistory = await historyStore.loadRaw(historyId, chunk.id) + const chunkMetadata = await ChunkStore.getLatestChunkMetadata(historyId) + const rawHistory = await historyStore.loadRaw(historyId, chunkMetadata.id) expect(JSON.parse(chunkContent)).to.deep.equal(rawHistory) // Unrelated entries from backedUpBlobs should be not cleared @@ -399,8 +399,8 @@ describe('backup script', function () { makeChunkKey(historyId, 0) ) const chunkContent = await text(chunkStream.pipe(createGunzip())) - const chunk = await ChunkStore.loadLatestRaw(historyId) - const rawHistory = await historyStore.loadRaw(historyId, chunk.id) + const chunkMetadata = await ChunkStore.getLatestChunkMetadata(historyId) + const rawHistory = await historyStore.loadRaw(historyId, chunkMetadata.id) expect(JSON.parse(chunkContent)).to.deep.equal(rawHistory) // Verify that the demoted global blob was backed up diff --git a/services/history-v1/test/acceptance/js/storage/chunk_store.test.js b/services/history-v1/test/acceptance/js/storage/chunk_store.test.js index 60710b2407..48c142817d 100644 --- a/services/history-v1/test/acceptance/js/storage/chunk_store.test.js +++ b/services/history-v1/test/acceptance/js/storage/chunk_store.test.js @@ -120,8 +120,9 @@ describe('chunkStore', function () { }) it('records the correct metadata in db readOnly=false', async function () { - const raw = await chunkStore.loadLatestRaw(projectId) - expect(raw).to.deep.include({ + const chunkMetadata = + await chunkStore.getLatestChunkMetadata(projectId) + expect(chunkMetadata).to.deep.include({ startVersion: 0, endVersion: 2, endTimestamp: lastChangeTimestamp, @@ -129,10 +130,11 @@ describe('chunkStore', function () { }) it('records the correct metadata in db readOnly=true', async function () { - const raw = await chunkStore.loadLatestRaw(projectId, { - readOnly: true, - }) - expect(raw).to.deep.include({ + const chunkMetadata = await chunkStore.getLatestChunkMetadata( + projectId, + { readOnly: true } + ) + expect(chunkMetadata).to.deep.include({ startVersion: 0, endVersion: 2, endTimestamp: lastChangeTimestamp,