From 68d4a29f9efcc720d80e0f0127f4019efffeda1e Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 27 Nov 2024 13:41:19 +0000 Subject: [PATCH] Merge pull request #22170 from overleaf/bg-history-v1-copy-blob add copyBlob support to history-v1 GitOrigin-RevId: 797ea66c37ca938fc906c4dff7bb1c8bf14c031e --- .../history-v1/api/controllers/projects.js | 25 +++++ services/history-v1/api/swagger/projects.js | 36 +++++++ .../storage/lib/blob_store/index.js | 25 +++++ .../acceptance/js/api/project_blobs.test.js | 101 ++++++++++++++++++ .../acceptance/js/storage/blob_store.test.js | 24 +++++ 5 files changed, 211 insertions(+) diff --git a/services/history-v1/api/controllers/projects.js b/services/history-v1/api/controllers/projects.js index 0370872348..1aba3012f3 100644 --- a/services/history-v1/api/controllers/projects.js +++ b/services/history-v1/api/controllers/projects.js @@ -222,6 +222,30 @@ async function getProjectBlob(req, res, next) { } } +async function copyProjectBlob(req, res, next) { + const sourceProjectId = req.swagger.params.copyFrom.value + const targetProjectId = req.swagger.params.project_id.value + const blobHash = req.swagger.params.hash.value + // Check that blob exists in source project + const sourceBlobStore = new BlobStore(sourceProjectId) + const targetBlobStore = new BlobStore(targetProjectId) + const [sourceBlob, targetBlob] = await Promise.all([ + sourceBlobStore.getBlob(blobHash), + targetBlobStore.getBlob(blobHash), + ]) + if (!sourceBlob) { + return render.notFound(res) + } + // Exit early if the blob exists in the target project. + // This will also catch global blobs, which always exist. + if (targetBlob) { + return res.status(HTTPStatus.NO_CONTENT).end() + } + // Otherwise, copy blob from source project to target project + await sourceBlobStore.copyBlob(sourceBlob, targetProjectId) + res.status(HTTPStatus.CREATED).end() +} + async function getSnapshotAtVersion(projectId, version) { const chunk = await chunkStore.loadAtVersion(projectId, version) const snapshot = chunk.getSnapshot() @@ -247,4 +271,5 @@ module.exports = { deleteProject: expressify(deleteProject), createProjectBlob: expressify(createProjectBlob), getProjectBlob: expressify(getProjectBlob), + copyProjectBlob: expressify(copyProjectBlob), } diff --git a/services/history-v1/api/swagger/projects.js b/services/history-v1/api/swagger/projects.js index 600e864f19..30f6759d6f 100644 --- a/services/history-v1/api/swagger/projects.js +++ b/services/history-v1/api/swagger/projects.js @@ -134,6 +134,42 @@ exports.paths = { }, }, }, + post: { + 'x-swagger-router-controller': 'projects', + operationId: 'copyProjectBlob', + tags: ['Project'], + description: + 'Copies a blob from a source project to a target project when duplicating a project', + parameters: [ + { + name: 'project_id', + in: 'path', + description: 'target project id', + required: true, + type: 'string', + }, + { + name: 'hash', + in: 'path', + description: 'Hexadecimal SHA-1 hash', + required: true, + type: 'string', + pattern: Blob.HEX_HASH_RX_STRING, + }, + { + name: 'copyFrom', + in: 'query', + description: 'source project id', + required: false, + type: 'string', + }, + ], + responses: { + 201: { + description: 'Created', + }, + }, + }, }, '/projects/{project_id}/latest/content': { get: { diff --git a/services/history-v1/storage/lib/blob_store/index.js b/services/history-v1/storage/lib/blob_store/index.js index c4151637c3..309f00f797 100644 --- a/services/history-v1/storage/lib/blob_store/index.js +++ b/services/history-v1/storage/lib/blob_store/index.js @@ -390,6 +390,31 @@ class BlobStore { const blob = await this.backend.findBlob(this.projectId, hash) return blob } + + /** + * Copy an existing sourceBlob in this project to a target project. + * @param {Blob} sourceBlob + * @param {string} targetProjectId + * @return {Promise} + */ + async copyBlob(sourceBlob, targetProjectId) { + assert.instance(sourceBlob, Blob, 'bad sourceBlob') + assert.projectId(targetProjectId, 'bad targetProjectId') + const hash = sourceBlob.getHash() + const sourceProjectId = this.projectId + const { bucket, key: sourceKey } = getBlobLocation(sourceProjectId, hash) + const destKey = makeProjectKey(targetProjectId, hash) + logger.debug({ sourceProjectId, targetProjectId, hash }, 'copyBlob started') + try { + await persistor.copyObject(bucket, sourceKey, destKey) + await this.backend.insertBlob(targetProjectId, sourceBlob) + } finally { + logger.debug( + { sourceProjectId, targetProjectId, hash }, + 'copyBlob finished' + ) + } + } } module.exports = { diff --git a/services/history-v1/test/acceptance/js/api/project_blobs.test.js b/services/history-v1/test/acceptance/js/api/project_blobs.test.js index 2a3eb806b1..ce274a2f4a 100644 --- a/services/history-v1/test/acceptance/js/api/project_blobs.test.js +++ b/services/history-v1/test/acceptance/js/api/project_blobs.test.js @@ -10,6 +10,11 @@ const testFiles = require('../storage/support/test_files') const testServer = require('./support/test_server') const { expectHttpError } = require('./support/expect_response') +const { globalBlobs } = require('../../../../storage/lib/mongodb.js') +const { + loadGlobalBlobs, +} = require('../../../../storage/lib/blob_store/index.js') + describe('Project blobs API', function () { const projectId = '123' @@ -119,5 +124,101 @@ describe('Project blobs API', function () { ) expect(response.status).to.equal(HTTPStatus.UNAUTHORIZED) }) + + it('copies the blob to another project', async function () { + const targetProjectId = '456' + const targetClient = + await testServer.createClientForProject(targetProjectId) + const targetToken = testServer.createTokenForProject(targetProjectId) + const url = new URL( + testServer.url( + `/api/projects/${targetProjectId}/blobs/${testFiles.HELLO_TXT_HASH}` + ) + ) + url.searchParams.append('copyFrom', projectId) + + const response = await fetch(url, { + method: 'POST', + headers: { Authorization: `Bearer ${targetToken}` }, + }) + expect(response.status).to.equal(HTTPStatus.CREATED) + + const newBlobResponse = await targetClient.apis.Project.getProjectBlob({ + project_id: targetProjectId, + hash: testFiles.HELLO_TXT_HASH, + }) + const newBlobResponseText = await newBlobResponse.data.text() + expect(newBlobResponseText).to.equal(fileContents.toString()) + }) + + it('skips copying a blob to another project if it already exists', async function () { + const targetProjectId = '456' + const targetClient = + await testServer.createClientForProject(targetProjectId) + const targetToken = testServer.createTokenForProject(targetProjectId) + + const fileContents = await fs.promises.readFile( + testFiles.path('hello.txt') + ) + const uploadResponse = await fetch( + testServer.url( + `/api/projects/${targetProjectId}/blobs/${testFiles.HELLO_TXT_HASH}` + ), + { + method: 'PUT', + headers: { Authorization: `Bearer ${targetToken}` }, + body: fileContents, + } + ) + expect(uploadResponse.ok).to.be.true + + const url = new URL( + testServer.url( + `/api/projects/${targetProjectId}/blobs/${testFiles.HELLO_TXT_HASH}` + ) + ) + url.searchParams.append('copyFrom', projectId) + + const response = await fetch(url, { + method: 'POST', + headers: { Authorization: `Bearer ${targetToken}` }, + }) + expect(response.status).to.equal(HTTPStatus.NO_CONTENT) + + const newBlobResponse = await targetClient.apis.Project.getProjectBlob({ + project_id: targetProjectId, + hash: testFiles.HELLO_TXT_HASH, + }) + const newBlobResponseText = await newBlobResponse.data.text() + expect(newBlobResponseText).to.equal(fileContents.toString()) + }) + }) + + describe('with a global blob', async function () { + before(async function () { + await globalBlobs.insertOne({ + _id: testFiles.STRING_A_HASH, + byteLength: 1, + stringLength: 1, + }) + await loadGlobalBlobs() + }) + + it('does not copy global blobs', async function () { + const targetProjectId = '456' + const targetToken = testServer.createTokenForProject(targetProjectId) + const url = new URL( + testServer.url( + `/api/projects/${targetProjectId}/blobs/${testFiles.STRING_A_HASH}` + ) + ) + url.searchParams.append('copyFrom', projectId) + + const response = await fetch(url, { + method: 'POST', + headers: { Authorization: `Bearer ${targetToken}` }, + }) + expect(response.status).to.equal(HTTPStatus.NO_CONTENT) + }) }) }) diff --git a/services/history-v1/test/acceptance/js/storage/blob_store.test.js b/services/history-v1/test/acceptance/js/storage/blob_store.test.js index f61c940356..e13530d1bf 100644 --- a/services/history-v1/test/acceptance/js/storage/blob_store.test.js +++ b/services/history-v1/test/acceptance/js/storage/blob_store.test.js @@ -478,6 +478,30 @@ describe('BlobStore', function () { expect(content).to.equal(globalBlobString) }) }) + + describe('copyBlob method', function () { + it('copies a binary blob to another project', async function () { + const testFile = 'graph.png' + const originalHash = testFiles.GRAPH_PNG_HASH + const insertedBlob = await blobStore.putFile(testFiles.path(testFile)) + await blobStore.copyBlob(insertedBlob, scenario.projectId2) + const copiedBlob = await blobStore2.getBlob(originalHash) + expect(copiedBlob.getHash()).to.equal(originalHash) + expect(copiedBlob.getByteLength()).to.equal( + insertedBlob.getByteLength() + ) + expect(copiedBlob.getStringLength()).to.be.null + }) + + it('copies a text blob to another project', async function () { + const insertedBlob = await blobStore.putString(helloWorldString) + await blobStore.copyBlob(insertedBlob, scenario.projectId2) + const copiedBlob = await blobStore2.getBlob(helloWorldHash) + expect(copiedBlob.getHash()).to.equal(helloWorldHash) + const content = await blobStore2.getString(helloWorldHash) + expect(content).to.equal(helloWorldString) + }) + }) }) }