Merge pull request #16368 from overleaf/em-remove-project-archiver-code

Remove project-archiver related code GitOrigin-RevId: b3331033658c14a7c4d8745fd7085cd7c65c94bc
2026-11-14 22:23:42 +01:00 · 2024-01-05 09:03:21 +00:00
parent 4a8a811cc1
commit b0f3f00c43
5 changed files with 0 additions and 204 deletions
@@ -1,134 +0,0 @@
-const Settings = require('@overleaf/settings')
-const { fetchJson } = require('@overleaf/fetch-utils')
-const { waitForDb } = require('../app/src/infrastructure/mongodb')
-const { promiseMapWithLimit } = require('@overleaf/promise-utils')
-const { getHardDeletedProjectIds } = require('./delete_orphaned_data_helper')
-const TpdsUpdateSender = require('../app/src/Features/ThirdPartyDataStore/TpdsUpdateSender')
-const { promisify } = require('util')
-const { ObjectId } = require('mongodb')
-const sleep = promisify(setTimeout)
-
-const START_OFFSET = process.env.START_OFFSET
-
-const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 1000
-const DRY_RUN = process.env.DRY_RUN !== 'false'
-const READ_CONCURRENCY_SECONDARY =
-  parseInt(process.env.READ_CONCURRENCY_SECONDARY, 10) || 1000
-const READ_CONCURRENCY_PRIMARY =
-  parseInt(process.env.READ_CONCURRENCY_PRIMARY, 10) || 500
-const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
-
-const LET_USER_DOUBLE_CHECK_INPUTS_FOR =
-  parseInt(process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR, 10) || 10 * 1000
-
-async function main() {
-  await letUserDoubleCheckInputs()
-  await waitForDb()
-
-  let processed = 0
-  let hardDeleted = 0
-  let pageToken = ''
-  let startOffset = START_OFFSET
-  while (pageToken !== undefined) {
-    const url = new URL(`${Settings.apis.project_archiver.url}/project/list`)
-    url.searchParams.append('pageToken', pageToken)
-    url.searchParams.append('startOffset', startOffset)
-    const { nextPageToken, entries } = await fetchJson(url)
-    pageToken = nextPageToken
-    startOffset = undefined
-
-    hardDeleted += await processBatch(entries)
-    processed += entries.length
-    console.log(
-      'processed:',
-      processed.toString().padStart(10, '0'),
-      'hard deleted:',
-      hardDeleted.toString().padStart(10, '0'),
-      'nextPageToken:',
-      nextPageToken,
-      'START_OFFSET:',
-      entries.pop()?.prefix
-    )
-  }
-}
-
-async function processBatch(entries) {
-  const projectIdToPrefix = new Map()
-  for (const { prefix, projectId } of entries) {
-    const prefixes = projectIdToPrefix.get(projectId) || []
-    prefixes.push(prefix)
-    projectIdToPrefix.set(projectId, prefixes)
-  }
-  const projectIds = Array.from(projectIdToPrefix.keys()).map(
-    id => new ObjectId(id)
-  )
-  const projectsWithOrphanedArchive = await getHardDeletedProjectIds({
-    projectIds,
-    READ_CONCURRENCY_PRIMARY,
-    READ_CONCURRENCY_SECONDARY,
-  })
-
-  await promiseMapWithLimit(
-    WRITE_CONCURRENCY,
-    projectsWithOrphanedArchive.flatMap(id =>
-      projectIdToPrefix.get(id.toString())
-    ),
-    hardDeleteProjectArchiverData
-  )
-  return projectsWithOrphanedArchive.length
-}
-
-async function hardDeleteProjectArchiverData(prefix) {
-  console.log(`Destroying hard deleted project archive at '${prefix}/'`)
-  if (DRY_RUN) return
-
-  for (let i = 0; i < 10; i++) {
-    await sleep(1000 * i)
-    try {
-      const ok = await TpdsUpdateSender.promises.deleteProject({
-        projectId: encodeURIComponent(prefix),
-      })
-      if (ok) {
-        return
-      }
-    } catch (e) {
-      console.error(`deletion failed for '${prefix}/'`, e)
-    }
-  }
-  throw new Error(`deletion failed for '${prefix}/', check logs`)
-}
-
-async function letUserDoubleCheckInputs() {
-  console.error(
-    'Options:',
-    JSON.stringify(
-      {
-        BATCH_SIZE,
-        DRY_RUN,
-        LET_USER_DOUBLE_CHECK_INPUTS_FOR,
-        READ_CONCURRENCY_SECONDARY,
-        READ_CONCURRENCY_PRIMARY,
-        START_OFFSET,
-        WRITE_CONCURRENCY,
-      },
-      null,
-      2
-    )
-  )
-  console.error(
-    'Waiting for you to double check inputs for',
-    LET_USER_DOUBLE_CHECK_INPUTS_FOR,
-    'ms'
-  )
-  await sleep(LET_USER_DOUBLE_CHECK_INPUTS_FOR)
-}
-
-main()
-  .then(() => {
-    console.log('Done.')
-    process.exit(0)
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exit(1)
-  })