Merge pull request #30045 from overleaf/bg-object-persistor-make-list-directory-safer

Improve safety of object persistor

GitOrigin-RevId: bced9814de6613b388ca288a5f72cd42cff6c1d3

libraries/object-persistor/src/AbstractPersistor.js

@@ -177,4 +177,21 @@ module.exports = class AbstractPersistor {
       prefix,
     })
   }
+
+  /**
+   * List objects in a directory, returning key and size information.
+   *
+   * Suitable only for directories where the number of keys is known to be small.
+   *
+   * @param {string} location
+   * @param {string} prefix
+   * @returns {Promise<Array<{key: string, size: number}>>}
+   */
+  async listDirectoryStats(location, prefix) {
+    throw new NotImplementedError('method not implemented in persistor', {
+      method: 'listDirectoryStats',
+      location,
+      prefix,
+    })
+  }
 }

libraries/object-persistor/src/FSPersistor.js

@@ -196,7 +196,46 @@ module.exports = class FSPersistor extends AbstractPersistor {
 
   async listDirectoryKeys(location, name) {
     const fsPath = this._getFsPath(location, name)
-    return await this._listDirectory(fsPath)
+    const paths = await this._listDirectory(fsPath)
+
+    // Filter to only return files, not directories
+    const files = []
+    for (const path of paths) {
+      try {
+        const stat = await fsPromises.stat(path)
+        if (stat.isFile()) {
+          files.push(path)
+        }
+      } catch (err) {
+        // ignore files that may have just been deleted
+        if (err.code !== 'ENOENT') {
+          throw err
+        }
+      }
+    }
+    return files
+  }
+
+  async listDirectoryStats(location, name) {
+    const fsPath = this._getFsPath(location, name)
+    const paths = await this._listDirectory(fsPath)
+
+    // Filter to only return files, not directories, with their sizes
+    const stats = []
+    for (const path of paths) {
+      try {
+        const stat = await fsPromises.stat(path)
+        if (stat.isFile()) {
+          stats.push({ key: path, size: stat.size })
+        }
+      } catch (err) {
+        // ignore files that may have just been deleted
+        if (err.code !== 'ENOENT') {
+          throw err
+        }
+      }
+    }
+    return stats
   }
 
   async checkIfObjectExists(location, name) {

libraries/object-persistor/src/GcsPersistor.js

@@ -322,6 +322,17 @@ module.exports = class GcsPersistor extends AbstractPersistor {
     return files.map(file => file.name)
   }
 
+  async listDirectoryStats(bucketName, prefix) {
+    const files = await this.#listDirectory(
+      bucketName,
+      ensurePrefixIsDirectory(prefix)
+    )
+    return files.map(file => ({
+      key: file.name,
+      size: parseInt(file.metadata.size, 10),
+    }))
+  }
+
   async checkIfObjectExists(bucketName, key) {
     try {
       const [response] = await this.storage

libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js

@@ -417,10 +417,20 @@ class CachedPerProjectEncryptedS3Persistor {
    *
    * @param {string} bucketName
    * @param {string} path
-   * @return {Promise<ListDirectoryResult>}
+   * @return {Promise<string[]>}
    */
-  async listDirectory(bucketName, path) {
-    return await this.#parent.listDirectory(bucketName, path)
+  async listDirectoryKeys(bucketName, path) {
+    return await this.#parent.listDirectoryKeys(bucketName, path)
+  }
+
+  /**
+   *
+   * @param {string} bucketName
+   * @param {string} path
+   * @return {Promise<Array<{key: string, size: number}>>}
+   */
+  async listDirectoryStats(bucketName, path) {
+    return await this.#parent.listDirectoryStats(bucketName, path)
   }
 
   /**

libraries/object-persistor/src/S3Persistor.js

@@ -266,7 +266,7 @@ class S3Persistor extends AbstractPersistor {
    * @return {Promise<void>}
    */
   async deleteDirectory(bucketName, key, continuationToken) {
-    const { contents, response } = await this.listDirectory(
+    const { contents, response } = await this.#listDirectory(
       bucketName,
       key,
       continuationToken
@@ -309,7 +309,7 @@ class S3Persistor extends AbstractPersistor {
    * @param {string} [continuationToken]
    * @return {Promise<ListDirectoryResult>}
    */
-  async listDirectory(bucketName, key, continuationToken) {
+  async #listDirectory(bucketName, key, continuationToken) {
     let response
     const options = { Bucket: bucketName, Prefix: key }
     if (continuationToken) {
@@ -535,6 +535,59 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
+  /**
+   * @param {string} bucketName
+   * @param {string} prefix
+   * @return {Promise<Array<string>>}
+   */
+  async listDirectoryKeys(bucketName, prefix) {
+    const keys = []
+    let continuationToken
+
+    do {
+      const { contents, response } = await this.#listDirectory(
+        bucketName,
+        prefix,
+        continuationToken
+      )
+      keys.push(...contents.map(item => item.Key || ''))
+      continuationToken = response.IsTruncated
+        ? response.NextContinuationToken
+        : undefined
+    } while (continuationToken)
+
+    return keys
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} prefix
+   * @return {Promise<Array<{key: string, size: number}>>}
+   */
+  async listDirectoryStats(bucketName, prefix) {
+    const stats = []
+    let continuationToken
+
+    do {
+      const { contents, response } = await this.#listDirectory(
+        bucketName,
+        prefix,
+        continuationToken
+      )
+      stats.push(
+        ...contents.map(item => ({
+          key: item.Key || '',
+          size: item.Size ?? -1,
+        }))
+      )
+      continuationToken = response.IsTruncated
+        ? response.NextContinuationToken
+        : undefined
+    } while (continuationToken)
+
+    return stats
+  }
+
   /**
    * @param {string} bucket
    * @return {S3Client}