From 27076c50cc379729eee7d6efe14dd1eb42470455 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 18 Nov 2024 17:08:59 +0100 Subject: [PATCH] Merge pull request #21670 from overleaf/jpa-mongo-backend-types [history-v1] add types to mongo BlobStore backend GitOrigin-RevId: 7d91074eaa781904f7f3b56390aacee1800a7f67 --- libraries/overleaf-editor-core/lib/blob.js | 2 +- .../overleaf-editor-core/lib/change_note.js | 2 +- libraries/overleaf-editor-core/lib/file.js | 2 +- .../lib/file_data/index.js | 2 +- .../lib/v2_doc_versions.js | 2 +- .../storage/lib/blob_store/mongo.js | 51 ++++++++++++++++--- .../history-v1/storage/lib/project_archive.js | 2 +- .../acceptance/js/storage/blob_store.test.js | 5 ++ 8 files changed, 56 insertions(+), 12 deletions(-) diff --git a/libraries/overleaf-editor-core/lib/blob.js b/libraries/overleaf-editor-core/lib/blob.js index 49e23fb862..7f1b7bb042 100644 --- a/libraries/overleaf-editor-core/lib/blob.js +++ b/libraries/overleaf-editor-core/lib/blob.js @@ -94,7 +94,7 @@ class Blob { /** * Utf-8 length of the blob content, if it appears to be valid UTF-8. - * @return {?number} + * @return {number|undefined} */ getStringLength() { return this.stringLength diff --git a/libraries/overleaf-editor-core/lib/change_note.js b/libraries/overleaf-editor-core/lib/change_note.js index 94bd344130..37042953fd 100644 --- a/libraries/overleaf-editor-core/lib/change_note.js +++ b/libraries/overleaf-editor-core/lib/change_note.js @@ -10,7 +10,7 @@ const Change = require('./change') class ChangeNote { /** * @param {number} baseVersion the new base version for the change - * @param {?Change} change + * @param {Change} [change] */ constructor(baseVersion, change) { assert.integer(baseVersion, 'bad baseVersion') diff --git a/libraries/overleaf-editor-core/lib/file.js b/libraries/overleaf-editor-core/lib/file.js index 2b7117eef3..b5321c39b2 100644 --- a/libraries/overleaf-editor-core/lib/file.js +++ b/libraries/overleaf-editor-core/lib/file.js @@ -95,7 +95,7 @@ class File { /** * @param {number} byteLength - * @param {number?} stringLength + * @param {number} [stringLength] * @param {Object} [metadata] * @return {File} */ diff --git a/libraries/overleaf-editor-core/lib/file_data/index.js b/libraries/overleaf-editor-core/lib/file_data/index.js index e7ab920de9..a6ae574a26 100644 --- a/libraries/overleaf-editor-core/lib/file_data/index.js +++ b/libraries/overleaf-editor-core/lib/file_data/index.js @@ -47,7 +47,7 @@ class FileData { /** @see File.createHollow * @param {number} byteLength - * @param {number|null} stringLength + * @param {number} [stringLength] */ static createHollow(byteLength, stringLength) { if (stringLength == null) { diff --git a/libraries/overleaf-editor-core/lib/v2_doc_versions.js b/libraries/overleaf-editor-core/lib/v2_doc_versions.js index 4d313bb2dd..2cfada5b2a 100644 --- a/libraries/overleaf-editor-core/lib/v2_doc_versions.js +++ b/libraries/overleaf-editor-core/lib/v2_doc_versions.js @@ -27,7 +27,7 @@ class V2DocVersions { } /** - * @return {?RawV2DocVersions} + * @return {RawV2DocVersions|null} */ toRaw() { if (!this.data) return null diff --git a/services/history-v1/storage/lib/blob_store/mongo.js b/services/history-v1/storage/lib/blob_store/mongo.js index 6bd516addb..665456401e 100644 --- a/services/history-v1/storage/lib/blob_store/mongo.js +++ b/services/history-v1/storage/lib/blob_store/mongo.js @@ -1,3 +1,4 @@ +// @ts-check /** * Mongo backend for the blob store. * @@ -15,7 +16,7 @@ */ const { Blob } = require('overleaf-editor-core') -const { ObjectId, Binary } = require('mongodb') +const { ObjectId, Binary, MongoError } = require('mongodb') const assert = require('../assert') const mongodb = require('../mongodb') @@ -24,6 +25,7 @@ const DUPLICATE_KEY_ERROR_CODE = 11000 /** * Set up the data structures for a given project. + * @param {string} projectId */ async function initialize(projectId) { assert.mongoId(projectId, 'bad projectId') @@ -33,14 +35,18 @@ async function initialize(projectId) { blobs: {}, }) } catch (err) { - if (err.code !== DUPLICATE_KEY_ERROR_CODE) { - throw err + if (err instanceof MongoError && err.code === DUPLICATE_KEY_ERROR_CODE) { + return // ignore already initialized case } + throw err } } /** * Return blob metadata for the given project and hash. + * @param {string} projectId + * @param {string} hash + * @return {Promise} */ async function findBlob(projectId, hash) { assert.mongoId(projectId, 'bad projectId') @@ -69,6 +75,9 @@ async function findBlob(projectId, hash) { /** * Search in the sharded collection for blob metadata + * @param {string} projectId + * @param {string} hash + * @return {Promise} */ async function findBlobSharded(projectId, hash) { const [shard, bucket] = getShardedBucket(hash) @@ -81,11 +90,15 @@ async function findBlobSharded(projectId, hash) { return null } const record = result.blobs.find(blob => blob.h.toString('hex') === hash) + if (!record) return null return recordToBlob(record) } /** * Read multiple blob metadata records by hexadecimal hashes. + * @param {string} projectId + * @param {Array} hashes + * @return {Promise>} */ async function findBlobs(projectId, hashes) { assert.mongoId(projectId, 'bad projectId') @@ -135,6 +148,9 @@ async function findBlobs(projectId, hashes) { /** * Search in the sharded collection for blob metadata. + * @param {string} projectId + * @param {Set} hashSet + * @return {Promise>} */ async function findBlobsSharded(projectId, hashSet) { // Build a map of buckets by shard key @@ -183,6 +199,8 @@ async function findBlobsSharded(projectId, hashSet) { /** * Add a blob's metadata to the blobs collection after it has been uploaded. + * @param {string} projectId + * @param {Blob} blob */ async function insertBlob(projectId, blob) { assert.mongoId(projectId, 'bad projectId') @@ -208,6 +226,10 @@ async function insertBlob(projectId, blob) { /** * Add a blob's metadata to the sharded blobs collection. + * @param {string} projectId + * @param {string} hash + * @param {Record} record + * @return {Promise} */ async function insertRecordSharded(projectId, hash, record) { const [shard, bucket] = getShardedBucket(hash) @@ -221,6 +243,7 @@ async function insertRecordSharded(projectId, hash, record) { /** * Delete all blobs for a given project. + * @param {string} projectId */ async function deleteBlobs(projectId) { assert.mongoId(projectId, 'bad projectId') @@ -228,12 +251,15 @@ async function deleteBlobs(projectId) { const minShardedId = makeShardedId(projectId, '0') const maxShardedId = makeShardedId(projectId, 'f') await mongodb.shardedBlobs.deleteMany({ + // @ts-ignore We are using a custom _id here. _id: { $gte: minShardedId, $lte: maxShardedId }, }) } /** * Return the Mongo path to the bucket for the given hash. + * @param {string} hash + * @return {string} */ function getBucket(hash) { return `blobs.${hash.slice(0, 3)}` @@ -242,6 +268,8 @@ function getBucket(hash) { /** * Return the shard key and Mongo path to the bucket for the given hash in the * sharded collection. + * @param {string} hash + * @return {[string, string]} */ function getShardedBucket(hash) { const shard = hash.slice(0, 1) @@ -251,13 +279,25 @@ function getShardedBucket(hash) { /** * Create an _id key for the sharded collection. + * @param {string} projectId + * @param {string} shard + * @return {Binary} */ function makeShardedId(projectId, shard) { return new Binary(Buffer.from(`${projectId}0${shard}`, 'hex')) } +/** + * @typedef {Object} Record + * @property {Binary} h + * @property {number} b + * @property {number} [s] + */ + /** * Return the Mongo record for the given blob. + * @param {Blob} blob + * @return {Record} */ function blobToRecord(blob) { const hash = blob.getHash() @@ -272,11 +312,10 @@ function blobToRecord(blob) { /** * Create a blob from the given Mongo record. + * @param {Record} record + * @return {Blob} */ function recordToBlob(record) { - if (record == null) { - return - } return new Blob(record.h.toString('hex'), record.b, record.s) } diff --git a/services/history-v1/storage/lib/project_archive.js b/services/history-v1/storage/lib/project_archive.js index 0d70fea5ea..8a8e93f1c9 100644 --- a/services/history-v1/storage/lib/project_archive.js +++ b/services/history-v1/storage/lib/project_archive.js @@ -49,7 +49,7 @@ class ProjectArchive { /** * @constructor * @param {Snapshot} snapshot - * @param {?number} timeout in ms + * @param {number} [timeout] in ms * @classdesc * Writes the project snapshot to a zip file. */ diff --git a/services/history-v1/test/acceptance/js/storage/blob_store.test.js b/services/history-v1/test/acceptance/js/storage/blob_store.test.js index 4fd29ac103..70b45a49c4 100644 --- a/services/history-v1/test/acceptance/js/storage/blob_store.test.js +++ b/services/history-v1/test/acceptance/js/storage/blob_store.test.js @@ -88,6 +88,11 @@ describe('BlobStore', function () { await blobStore2.initialize() }) + it('can initialize a project again without throwing an error', async function () { + await blobStore.initialize() + await blobStore2.initialize() + }) + it('can store and fetch string content', async function () { function checkBlob(blob) { expect(blob.getHash()).to.equal(helloWorldHash)