Merge pull request #21681 from overleaf/jpa-back-fill-file-hash

[history-v1] add script for back filling hash into project file-tree

GitOrigin-RevId: 70ab8c72f3eea1062708e4471b4ad1c60317ad26

libraries/mongo-utils/batchedUpdate.js

@@ -277,6 +277,7 @@ function batchedUpdateWithResultHandling(
 }
 
 module.exports = {
+  READ_PREFERENCE_SECONDARY,
   batchedUpdate,
   batchedUpdateWithResultHandling,
 }

libraries/object-persistor/src/AbstractPersistor.js

@@ -35,12 +35,12 @@ module.exports = class AbstractPersistor {
   /**
    * @param location
    * @param name
-   * @param {Object} opts
-   * @param {Number} opts.start
-   * @param {Number} opts.end
+   * @param {Object} [opts]
+   * @param {Number} [opts.start]
+   * @param {Number} [opts.end]
    * @return {Promise<NodeJS.ReadableStream>}
    */
-  async getObjectStream(location, name, opts) {
+  async getObjectStream(location, name, opts = {}) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'getObjectStream',
       location,

libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js

@@ -201,6 +201,7 @@ class PerProjectEncryptedS3Persistor extends S3Persistor {
         // Do not overwrite any objects if already created
         ifNoneMatch: '*',
         ssecOptions: await this.#getCurrentKeyEncryptionKey(projectFolder),
+        contentLength: 32,
       }
     )
     return new SSECOptions(dataEncryptionKey)
@@ -404,6 +405,7 @@ class CachedPerProjectEncryptedS3Persistor {
    * @param {Object} opts
    * @param {string} [opts.contentType]
    * @param {string} [opts.contentEncoding]
+   * @param {number} [opts.contentLength]
    * @param {'*'} [opts.ifNoneMatch]
    * @param {SSECOptions} [opts.ssecOptions]
    * @param {string} [opts.sourceMd5]

libraries/object-persistor/src/S3Persistor.js

@@ -85,6 +85,7 @@ class S3Persistor extends AbstractPersistor {
    * @param {Object} opts
    * @param {string} [opts.contentType]
    * @param {string} [opts.contentEncoding]
+   * @param {number} [opts.contentLength]
    * @param {'*'} [opts.ifNoneMatch]
    * @param {SSECOptions} [opts.ssecOptions]
    * @param {string} [opts.sourceMd5]
@@ -118,6 +119,9 @@ class S3Persistor extends AbstractPersistor {
       if (opts.contentEncoding) {
         uploadOptions.ContentEncoding = opts.contentEncoding
       }
+      if (opts.contentLength) {
+        uploadOptions.ContentLength = opts.contentLength
+      }
       if (opts.ifNoneMatch === '*') {
         uploadOptions.IfNoneMatch = '*'
       }
@@ -134,9 +138,15 @@ class S3Persistor extends AbstractPersistor {
         clientOptions.computeChecksums = true
       }
 
-      await this._getClientForBucket(bucketName, clientOptions)
-        .upload(uploadOptions, { partSize: this.settings.partSize })
-        .promise()
+      if (this.settings.disableMultiPartUpload) {
+        await this._getClientForBucket(bucketName, clientOptions)
+          .putObject(uploadOptions)
+          .promise()
+      } else {
+        await this._getClientForBucket(bucketName, clientOptions)
+          .upload(uploadOptions, { partSize: this.settings.partSize })
+          .promise()
+      }
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -150,7 +160,7 @@ class S3Persistor extends AbstractPersistor {
   /**
    * @param {string} bucketName
    * @param {string} key
-   * @param {Object} opts
+   * @param {Object} [opts]
    * @param {number} [opts.start]
    * @param {number} [opts.end]
    * @param {boolean} [opts.autoGunzip]