diff --git a/libraries/object-persistor/src/AbstractPersistor.js b/libraries/object-persistor/src/AbstractPersistor.js index 3352766655..519fc7f954 100644 --- a/libraries/object-persistor/src/AbstractPersistor.js +++ b/libraries/object-persistor/src/AbstractPersistor.js @@ -160,4 +160,21 @@ module.exports = class AbstractPersistor { name, }) } + + /** + * List objects in a directory, returning the full keys. + * + * Suitable only for directories where the number of keys is known to be small. + * + * @param {string} location + * @param {string} prefix + * @returns {Promise>} + */ + async listDirectoryKeys(location, prefix) { + throw new NotImplementedError('method not implemented in persistor', { + method: 'listDirectoryKeys', + location, + prefix, + }) + } } diff --git a/libraries/object-persistor/src/FSPersistor.js b/libraries/object-persistor/src/FSPersistor.js index 3bdfc8f07e..549a9a8c68 100644 --- a/libraries/object-persistor/src/FSPersistor.js +++ b/libraries/object-persistor/src/FSPersistor.js @@ -194,6 +194,11 @@ module.exports = class FSPersistor extends AbstractPersistor { } } + async listDirectoryKeys(location, name) { + const fsPath = this._getFsPath(location, name) + return await this._listDirectory(fsPath) + } + async checkIfObjectExists(location, name) { const fsPath = this._getFsPath(location, name) try { diff --git a/libraries/object-persistor/src/GcsPersistor.js b/libraries/object-persistor/src/GcsPersistor.js index 0485373ae5..c90978eff7 100644 --- a/libraries/object-persistor/src/GcsPersistor.js +++ b/libraries/object-persistor/src/GcsPersistor.js @@ -288,23 +288,25 @@ module.exports = class GcsPersistor extends AbstractPersistor { } while (query) } - async directorySize(bucketName, key) { - let files - const prefix = ensurePrefixIsDirectory(key) - + async #listDirectory(bucketName, prefix) { try { const [response] = await this.storage .bucket(bucketName) .getFiles({ prefix }) - files = response + return response } catch (err) { throw PersistorHelper.wrapError( err, 'failed to list objects in GCS', - { bucketName, key }, + { bucketName, prefix }, ReadError ) } + } + + async directorySize(bucketName, key) { + const prefix = ensurePrefixIsDirectory(key) + const files = await this.#listDirectory(bucketName, prefix) return files.reduce( (acc, file) => parseInt(file.metadata.size, 10) + acc, @@ -312,6 +314,14 @@ module.exports = class GcsPersistor extends AbstractPersistor { ) } + async listDirectoryKeys(bucketName, prefix) { + const files = await this.#listDirectory( + bucketName, + ensurePrefixIsDirectory(prefix) + ) + return files.map(file => file.name) + } + async checkIfObjectExists(bucketName, key) { try { const [response] = await this.storage diff --git a/services/web/.eslintrc.js b/services/web/.eslintrc.js index f8602a5b92..69b9bff98f 100644 --- a/services/web/.eslintrc.js +++ b/services/web/.eslintrc.js @@ -353,6 +353,7 @@ module.exports = { 'scripts/suspend_users.mjs', 'scripts/sync-user-entitlements/sync-user-entitlements.mjs', 'scripts/update_project_image_name.mjs', + 'scripts/user-export/analytics.mjs', 'scripts/user-export/fs.mjs', 'scripts/user-export/http.mjs', 'scripts/user-export/observer.mjs',