Support metadata when uploading objects

Add contentType and contentEncoding options to sendStream(). These
options will set the corresponding metadata on the object.

This changes the API. The fourth argument to sendStream() used to be the
source md5. Now, it's an options object and the source md5 is a property
of that object.

libraries/object-persistor/src/AbstractPersistor.js

@@ -10,12 +10,12 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  async sendStream(location, target, sourceStream, sourceMd5) {
+  async sendStream(location, target, sourceStream, opts = {}) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'sendStream',
       location,
       target,
-      sourceMd5
+      opts
     })
   }
 

libraries/object-persistor/src/FSPersistor.js

@@ -43,8 +43,9 @@ module.exports = class FSPersistor extends AbstractPersistor {
     }
   }
 
-  async sendStream(location, target, sourceStream, sourceMd5) {
+  async sendStream(location, target, sourceStream, opts = {}) {
     const fsPath = await this._writeStream(sourceStream)
+    let sourceMd5 = opts.sourceMd5
     if (!sourceMd5) {
       sourceMd5 = await FSPersistor._getFileMd5HashForPath(fsPath)
     }

libraries/object-persistor/src/GcsPersistor.js

@@ -39,7 +39,7 @@ module.exports = class GcsPersistor extends AbstractPersistor {
     return this.sendStream(bucketName, key, fs.createReadStream(fsPath))
   }
 
-  async sendStream(bucketName, key, readStream, sourceMd5) {
+  async sendStream(bucketName, key, readStream, opts = {}) {
     try {
       // egress from us to gcs
       const observeOptions = {
@@ -47,6 +47,7 @@ module.exports = class GcsPersistor extends AbstractPersistor {
         Metrics: this.settings.Metrics
       }
 
+      let sourceMd5 = opts.sourceMd5
       if (!sourceMd5) {
         // if there is no supplied md5 hash, we calculate the hash as the data passes through
         observeOptions.hash = 'md5'
@@ -61,9 +62,16 @@ module.exports = class GcsPersistor extends AbstractPersistor {
 
       if (sourceMd5) {
         writeOptions.validation = 'md5'
-        writeOptions.metadata = {
-          md5Hash: PersistorHelper.hexToBase64(sourceMd5)
-        }
+        writeOptions.metadata = writeOptions.metadata || {}
+        writeOptions.metadata.md5Hash = PersistorHelper.hexToBase64(sourceMd5)
+      }
+      if (opts.contentType) {
+        writeOptions.metadata = writeOptions.metadata || {}
+        writeOptions.metadata.contentType = opts.contentType
+      }
+      if (opts.contentEncoding) {
+        writeOptions.metadata = writeOptions.metadata || {}
+        writeOptions.metadata.contentEncoding = opts.contentEncoding
       }
 
       const uploadStream = this.storage

libraries/object-persistor/src/MigrationPersistor.js

@@ -172,12 +172,9 @@ module.exports = class MigrationPersistor extends AbstractPersistor {
         Logger.warn(err, 'error getting md5 hash from fallback persistor')
       }
 
-      await this.primaryPersistor.sendStream(
-        destBucket,
-        destKey,
-        stream,
+      await this.primaryPersistor.sendStream(destBucket, destKey, stream, {
         sourceMd5
-      )
+      })
     } catch (err) {
       const error = new WriteError(
         'unable to copy file to destination persistor',

libraries/object-persistor/src/S3Persistor.js

@@ -31,7 +31,7 @@ module.exports = class S3Persistor extends AbstractPersistor {
     return this.sendStream(bucketName, key, fs.createReadStream(fsPath))
   }
 
-  async sendStream(bucketName, key, readStream, sourceMd5) {
+  async sendStream(bucketName, key, readStream, opts = {}) {
     try {
       // egress from us to S3
       const observeOptions = {
@@ -50,11 +50,18 @@ module.exports = class S3Persistor extends AbstractPersistor {
         Body: observer
       }
 
+      if (opts.contentType) {
+        uploadOptions.ContentType = opts.contentType
+      }
+      if (opts.contentEncoding) {
+        uploadOptions.ContentEncoding = opts.contentEncoding
+      }
+
       // if we have an md5 hash, pass this to S3 to verify the upload - otherwise
       // we rely on the S3 client's checksum calculation to validate the upload
       const clientOptions = {}
-      if (sourceMd5) {
-        uploadOptions.ContentMD5 = PersistorHelper.hexToBase64(sourceMd5)
+      if (opts.sourceMd5) {
+        uploadOptions.ContentMD5 = PersistorHelper.hexToBase64(opts.sourceMd5)
       } else {
         clientOptions.computeChecksums = true
       }