[DocArchiveManager] destroy in small batches

Also add settings options for PARALLEL_JOBS and DESTROY_BATCH_SIZE.

services/docstore/app/js/DocArchiveManager.js

@@ -9,7 +9,8 @@ const RangeManager = require('./RangeManager')
 const PersistorManager = require('./PersistorManager')
 const pMap = require('p-map')
 
-const PARALLEL_JOBS = 5
+const PARALLEL_JOBS = settings.parallelArchiveJobs
+const DESTROY_BATCH_SIZE = settings.destroyBatchSize
 
 module.exports = {
   archiveAllDocs: callbackify(archiveAllDocs),
@@ -175,12 +176,15 @@ async function unarchiveDoc(projectId, docId) {
 }
 
 async function destroyAllDocs(projectId) {
-  const docs = await MongoManager.getProjectsDocs(
-    projectId,
-    { include_deleted: true },
-    { _id: 1 }
-  )
-  if (docs && docs.length) {
+  while (true) {
+    const docs = await MongoManager.getProjectsDocs(
+      projectId,
+      { include_deleted: true, limit: DESTROY_BATCH_SIZE },
+      { _id: 1 }
+    )
+    if (!docs || docs.length === 0) {
+      break
+    }
     await pMap(docs, (doc) => destroyDoc(projectId, doc._id), {
       concurrency: PARALLEL_JOBS
     })

services/docstore/app/js/MongoManager.js

@@ -57,11 +57,13 @@ module.exports = MongoManager = {
     if (!options.include_deleted) {
       query.deleted = { $ne: true }
     }
-    db.docs
-      .find(query, {
-        projection: filter
-      })
-      .toArray(callback)
+    const queryOptions = {
+      projection: filter
+    }
+    if (options.limit) {
+      queryOptions.limit = options.limit
+    }
+    db.docs.find(query, queryOptions).toArray(callback)
   },
 
   getArchivedProjectDocs(project_id, callback) {