In collect_paypal_past_due_invoice.js, iterate over each page instead of gathering data from all pages at first (#18414)

* Create `getPaginatedEndpointIterator` to iterate each page

* Create `waitMs` util, it will replace `slowCallback`

* Make `handleAPIError` async

* Make `isAccountUsingPaypal` async

* Make `attemptInvoiceCollection` async

* Make `attemptInvoicesCollection` async

* Use `await` instead of `new Promise`

* Remove unused callbackified `attemptInvoiceCollection`

* Run `attemptInvoiceCollection` for each page instead of gathering all pages in the beginning

* Add test on fetching multiple pages of invoice

GitOrigin-RevId: 2674b18c6ca5732b873fb2bc71b515909006f93d

services/web/scripts/recurly/collect_paypal_past_due_invoice.js

@@ -1,20 +1,19 @@
 const RecurlyWrapper = require('../../app/src/Features/Subscription/RecurlyWrapper')
-const async = require('async')
 const minimist = require('minimist')
 const logger = require('@overleaf/logger')
 
-const slowCallback =
+const waitMs =
   require.main === module
-    ? (callback, error, data) => setTimeout(() => callback(error, data), 80)
-    : (callback, error, data) => callback(error, data)
+    ? timeout => new Promise(resolve => setTimeout(() => resolve(), timeout))
+    : () => Promise.resolve()
 
 // NOTE: Errors are not propagated to the caller
-const handleAPIError = (source, id, error, callback) => {
+const handleAPIError = async (source, id, error) => {
   logger.warn(`Errors in ${source} with id=${id}`, error)
   if (typeof error === 'string' && error.match(/429$/)) {
-    return setTimeout(callback, 1000 * 60 * 5)
+    return waitMs(1000 * 60 * 5)
   }
-  slowCallback(callback)
+  await waitMs(80)
 }
 
 /**
@@ -25,100 +24,95 @@ const handleAPIError = (source, id, error, callback) => {
  * }>}
  */
 const main = async () => {
-  const attemptInvoiceCollection = (invoice, callback) => {
-    isAccountUsingPaypal(invoice, (error, isPaypal) => {
-      if (error || !isPaypal) {
-        return callback(error)
-      }
-      const accountId = invoice.account.url.match(/accounts\/(.*)/)[1]
-      if (USERS_COLLECTED.indexOf(accountId) > -1) {
-        logger.warn(`Skipping duplicate user ${accountId}`)
-        return callback()
-      }
-      INVOICES_COLLECTED.push(invoice.invoice_number)
-      USERS_COLLECTED.push(accountId)
-      if (DRY_RUN) {
-        return callback()
-      }
-      RecurlyWrapper.attemptInvoiceCollection(
-        invoice.invoice_number,
-        (error, response) => {
-          if (error) {
-            return handleAPIError(
-              'attemptInvoiceCollection',
-              invoice.invoice_number,
-              error,
-              callback
-            )
-          }
-          INVOICES_COLLECTED_SUCCESS.push(invoice.invoice_number)
-          slowCallback(callback, null)
-        }
-      )
-    })
-  }
+  const attemptInvoiceCollection = async invoice => {
+    const isPaypal = await isAccountUsingPaypal(invoice)
 
-  const isAccountUsingPaypal = (invoice, callback) => {
+    if (!isPaypal) {
+      return
+    }
     const accountId = invoice.account.url.match(/accounts\/(.*)/)[1]
-    RecurlyWrapper.getBillingInfo(accountId, (error, response) => {
-      if (error) {
-        return handleAPIError('billing info', accountId, error, callback)
-      }
-      if (response.billing_info.paypal_billing_agreement_id) {
-        return slowCallback(callback, null, true)
-      }
-      slowCallback(callback, null, false)
-    })
+    if (USERS_COLLECTED.indexOf(accountId) > -1) {
+      logger.warn(`Skipping duplicate user ${accountId}`)
+      return
+    }
+    INVOICES_COLLECTED.push(invoice.invoice_number)
+    USERS_COLLECTED.push(accountId)
+    if (DRY_RUN) {
+      return
+    }
+    try {
+      await RecurlyWrapper.promises.attemptInvoiceCollection(
+        invoice.invoice_number
+      )
+      INVOICES_COLLECTED_SUCCESS.push(invoice.invoice_number)
+      await waitMs(80)
+    } catch (error) {
+      return handleAPIError(
+        'attemptInvoiceCollection',
+        invoice.invoice_number,
+        error
+      )
+    }
   }
 
-  const attemptInvoicesCollection = callback => {
-    RecurlyWrapper.getPaginatedEndpoint(
-      'invoices',
-      { state: 'past_due' },
-      (error, invoices) => {
-        logger.info('invoices', invoices?.length)
-        if (error) {
-          return callback(error)
-        }
-        async.eachSeries(invoices, attemptInvoiceCollection, callback)
-      }
-    )
+  const isAccountUsingPaypal = async invoice => {
+    const accountId = invoice.account.url.match(/accounts\/(.*)/)[1]
+    try {
+      const response = await RecurlyWrapper.promises.getBillingInfo(accountId)
+      await waitMs(80)
+      return !!response.billing_info.paypal_billing_agreement_id
+    } catch (error) {
+      return handleAPIError('billing info', accountId, error)
+    }
   }
+
+  const attemptInvoicesCollection = async () => {
+    let getPage = await RecurlyWrapper.promises.getPaginatedEndpointIterator(
+      'invoices',
+      { state: 'past_due' }
+    )
+
+    while (getPage) {
+      const { items, getNextPage } = await getPage()
+      logger.info('invoices', items?.length)
+      for (const invoice of items) {
+        await attemptInvoiceCollection(invoice)
+      }
+      getPage = getNextPage
+    }
+  }
+
   const argv = minimist(process.argv.slice(2))
   const DRY_RUN = argv.n !== undefined
   const INVOICES_COLLECTED = []
   const INVOICES_COLLECTED_SUCCESS = []
   const USERS_COLLECTED = []
 
-  return new Promise((resolve, reject) => {
-    attemptInvoicesCollection(error => {
-      logger.info(
-        `DONE (DRY_RUN=${DRY_RUN}). ${INVOICES_COLLECTED.length} invoices collection attempts for ${USERS_COLLECTED.length} users. ${INVOICES_COLLECTED_SUCCESS.length} successful collections`
-      )
-      console.dir(
-        {
-          INVOICES_COLLECTED,
-          INVOICES_COLLECTED_SUCCESS,
-          USERS_COLLECTED,
-        },
-        { maxArrayLength: null }
-      )
+  try {
+    await attemptInvoicesCollection()
 
-      if (error) {
-        reject(error)
-      }
+    if (INVOICES_COLLECTED_SUCCESS.length === 0) {
+      throw new Error('No invoices collected')
+    }
 
-      if (INVOICES_COLLECTED_SUCCESS.length === 0) {
-        reject(new Error('No invoices collected'))
-      }
-
-      resolve({
+    return {
+      INVOICES_COLLECTED,
+      INVOICES_COLLECTED_SUCCESS,
+      USERS_COLLECTED,
+    }
+  } finally {
+    logger.info(
+      `DONE (DRY_RUN=${DRY_RUN}). ${INVOICES_COLLECTED.length} invoices collection attempts for ${USERS_COLLECTED.length} users. ${INVOICES_COLLECTED_SUCCESS.length} successful collections`
+    )
+    console.dir(
+      {
         INVOICES_COLLECTED,
         INVOICES_COLLECTED_SUCCESS,
         USERS_COLLECTED,
-      })
-    })
-  })
+      },
+      { maxArrayLength: null }
+    )
+  }
 }
 
 if (require.main === module) {