diff --git a/services/history-v1/storage/lib/blob_store/index.js b/services/history-v1/storage/lib/blob_store/index.js index 89daf60d7b..ef52541e8c 100644 --- a/services/history-v1/storage/lib/blob_store/index.js +++ b/services/history-v1/storage/lib/blob_store/index.js @@ -317,6 +317,16 @@ class BlobStore { return blobs } + /** + * Retrieve all blobs associated with the project. + * @returns {Promise} A promise that resolves to an array of blobs. + */ + + async getProjectBlobs() { + const projectBlobs = await this.backend.getProjectBlobs(this.projectId) + return projectBlobs + } + /** * Delete all blobs that belong to the project. */ diff --git a/services/history-v1/storage/lib/blob_store/mongo.js b/services/history-v1/storage/lib/blob_store/mongo.js index 665456401e..276c7f097f 100644 --- a/services/history-v1/storage/lib/blob_store/mongo.js +++ b/services/history-v1/storage/lib/blob_store/mongo.js @@ -197,6 +197,55 @@ async function findBlobsSharded(projectId, hashSet) { return blobs } +/** + * Return metadata for all blobs in the given project + */ +async function getProjectBlobs(projectId) { + assert.mongoId(projectId, 'bad projectId') + + const result = await mongodb.blobs.findOne( + { _id: new ObjectId(projectId) }, + { projection: { _id: 0 } } + ) + + if (!result) { + return [] + } + + // Build blobs from the query results + const blobs = [] + for (const bucket of Object.values(result.blobs)) { + for (const record of bucket) { + blobs.push(recordToBlob(record)) + } + } + + // Look for all possible sharded blobs + + const minShardedId = makeShardedId(projectId, '0') + const maxShardedId = makeShardedId(projectId, 'f') + // @ts-ignore We are using a custom _id here. + const shardedRecords = mongodb.shardedBlobs.find( + { + _id: { $gte: minShardedId, $lte: maxShardedId }, + }, + { projection: { _id: 0 } } + ) + + for await (const shardedRecord of shardedRecords) { + if (shardedRecord.blobs == null) { + continue + } + for (const bucket of Object.values(shardedRecord.blobs)) { + for (const record of bucket) { + blobs.push(recordToBlob(record)) + } + } + } + + return blobs +} + /** * Add a blob's metadata to the blobs collection after it has been uploaded. * @param {string} projectId @@ -323,6 +372,7 @@ module.exports = { initialize, findBlob, findBlobs, + getProjectBlobs, insertBlob, deleteBlobs, } diff --git a/services/history-v1/storage/lib/blob_store/postgres.js b/services/history-v1/storage/lib/blob_store/postgres.js index 9e40c255da..fc41644138 100644 --- a/services/history-v1/storage/lib/blob_store/postgres.js +++ b/services/history-v1/storage/lib/blob_store/postgres.js @@ -53,6 +53,23 @@ async function findBlobs(projectId, hashes) { return blobs } +/** + * Return metadata for all blobs in the given project + */ +async function getProjectBlobs(projectId) { + projectId = parseInt(projectId, 10) + assert.integer(projectId, 'bad projectId') + + const records = await knex('project_blobs') + .select('hash_bytes', 'byte_length', 'string_length') + .where({ + project_id: projectId, + }) + + const blobs = records.map(recordToBlob) + return blobs +} + /** * Add a blob's metadata to the blobs table after it has been uploaded. */ @@ -108,6 +125,7 @@ module.exports = { initialize, findBlob, findBlobs, + getProjectBlobs, insertBlob, deleteBlobs, } diff --git a/services/history-v1/test/acceptance/js/storage/blob_store.test.js b/services/history-v1/test/acceptance/js/storage/blob_store.test.js index 70b45a49c4..3e4a19a204 100644 --- a/services/history-v1/test/acceptance/js/storage/blob_store.test.js +++ b/services/history-v1/test/acceptance/js/storage/blob_store.test.js @@ -259,6 +259,16 @@ describe('BlobStore', function () { testFiles.GRAPH_PNG_HASH, ]) }) + + it('getProjectBlobs() returns all blobs in the project', async function () { + const blobs = await blobStore.getProjectBlobs() + const hashes = blobs.map(blob => blob.getHash()) + expect(hashes).to.have.members([ + testFiles.HELLO_TXT_HASH, + testFiles.GRAPH_PNG_HASH, + helloWorldHash, + ]) + }) }) describe('two blob stores on different projects', function () { diff --git a/services/history-v1/test/acceptance/js/storage/blob_store_mongo.test.js b/services/history-v1/test/acceptance/js/storage/blob_store_mongo.test.js index 834a4c9ed8..d2695681ab 100644 --- a/services/history-v1/test/acceptance/js/storage/blob_store_mongo.test.js +++ b/services/history-v1/test/acceptance/js/storage/blob_store_mongo.test.js @@ -113,6 +113,15 @@ describe('BlobStore Mongo backend', function () { }) }) + describe('getProjectBlobs', function () { + it('returns all blobs for a given project', async function () { + const blobs = await mongoBackend.getProjectBlobs(projectId) + const obtainedHashes = blobs.map(blob => blob.getHash()) + const expectedHashes = hashes.abcd.concat(hashes[1234]) + expect(obtainedHashes).to.have.members(expectedHashes) + }) + }) + describe('deleteBlobs', function () { it('deletes all blobs for a given project', async function () { await mongoBackend.deleteBlobs(projectId)