[web] allow admins to clone projects with ranges and entire history (#32739)

* [web] add consistent aria-label to editing/reviewing toggle

* [docstore] add endpoint for getting all docs with ranges

* [history-v1] fix schema of chunkId when deleting old history chunk

* [web] skip duplicate project lookup for resolving rootDocPath

* [web] ignore new limits for root doc path when making debug copy

* [web] allow admins to clone projects with ranges and entire history

* [web] fix tests

* [history-v1] re-order params for cloning project

* [web] fix duplicate import of logger after merge

* [project-history] re-order params for cloning project history metadata

GitOrigin-RevId: 7fa35b4f90885dd453150a348d491ba0ec8de412

services/project-history/app/js/ErrorRecorder.js

@@ -86,6 +86,23 @@ async function recordSyncStart(projectId) {
   )
 }
 
+/**
+ * @param {string} sourceProjectId
+ * @param {string} targetProjectId
+ * @return {Promise<void>}
+ */
+async function cloneFailure(sourceProjectId, targetProjectId) {
+  const failure = await db.projectHistoryFailures.findOne(
+    { project_id: sourceProjectId.toString() },
+    { projection: { _id: 0, project_id: 0 } }
+  )
+  if (!failure) return
+  await db.projectHistoryFailures.insertOne({
+    ...failure,
+    project_id: targetProjectId.toString(),
+  })
+}
+
 /**
  * @param projectId
  */
@@ -238,6 +255,7 @@ async function getFailures() {
 const getFailedProjectsCb = callbackify(getFailedProjects)
 const getFailureRecordCb = callbackify(getFailureRecord)
 const getFailuresCb = callbackify(getFailures)
+const cloneFailureCb = callbackify(cloneFailure)
 const getLastFailureCb = callbackify(getLastFailure)
 const recordCb = callbackify(record)
 const clearErrorCb = callbackify(clearError)
@@ -245,6 +263,7 @@ const recordSyncStartCb = callbackify(recordSyncStart)
 const setForceDebugCb = callbackify(setForceDebug)
 
 export {
+  cloneFailureCb as cloneFailure,
   getFailedProjectsCb as getFailedProjects,
   getFailureRecordCb as getFailureRecord,
   getLastFailureCb as getLastFailure,
@@ -257,6 +276,7 @@ export {
 
 export const promises = {
   getFailedProjects,
+  cloneFailure,
   getFailureRecord,
   getLastFailure,
   getFailures,

services/project-history/app/js/HistoryStoreManager.js

@@ -17,7 +17,7 @@ import * as Errors from './Errors.js'
 import * as LocalFileWriter from './LocalFileWriter.js'
 import * as HashManager from './HashManager.js'
 import * as HistoryBlobTranslator from './HistoryBlobTranslator.js'
-import { promisifyMultiResult } from '@overleaf/promise-utils'
+import { callbackify, promisifyMultiResult } from '@overleaf/promise-utils'
 
 const HTTP_REQUEST_TIMEOUT = Settings.overleaf.history.requestTimeout
 
@@ -531,6 +531,18 @@ export function initializeProject(historyId, callback) {
   )
 }
 
+async function _cloneProject(sourceProjectId, targetProjectId, signal) {
+  return await fetchStream(
+    `${Settings.overleaf.history.host}/projects/${sourceProjectId}/clone`,
+    {
+      method: 'POST',
+      json: { targetProjectId },
+      ...getHistoryFetchOptions(),
+      signal,
+    }
+  )
+}
+
 export function deleteProject(projectId, callback) {
   _requestHistoryService(
     { method: 'DELETE', path: `projects/${projectId}` },
@@ -623,6 +635,8 @@ function _requestHistoryService(options, callback) {
   })
 }
 
+export const cloneProject = callbackify(_cloneProject)
+
 export const promises = {
   /** @type {(projectId: string, historyId: string) => Promise<{chunk: import('overleaf-editor-core/lib/types.js').RawChunk}>} */
   getMostRecentChunk: promisify(getMostRecentChunk),
@@ -640,4 +654,5 @@ export const promises = {
   createBlobForUpdate: promisify(createBlobForUpdate),
   initializeProject: promisify(initializeProject),
   deleteProject: promisify(deleteProject),
+  cloneProject: _cloneProject,
 }

services/project-history/app/js/HttpController.js

@@ -14,12 +14,114 @@ import * as LabelsManager from './LabelsManager.js'
 import * as HistoryApiManager from './HistoryApiManager.js'
 import * as RetryManager from './RetryManager.js'
 import * as FlushManager from './FlushManager.js'
-import { pipeline } from 'node:stream'
+import Stream, { pipeline } from 'node:stream'
 import { fetchNothing, RequestFailedError } from '@overleaf/fetch-utils'
 import { z, zz, parseReq } from '@overleaf/validation-tools'
+import { IncrementalResponse } from '@overleaf/stream-utils'
 
 const ONE_DAY_IN_SECONDS = 24 * 60 * 60
 
+const cloneProjectSchema = z.object({
+  body: z.object({
+    targetProjectId: z.string(),
+  }),
+
+  params: z.object({
+    project_id: z.string(),
+  }),
+})
+
+export function cloneProject(req, res) {
+  const {
+    params: { project_id: sourceProjectId },
+    body: { targetProjectId },
+  } = parseReq(req, cloneProjectSchema)
+  const incrResp = new IncrementalResponse({
+    res,
+    timeout: 10 * 60_000 - 5_000,
+    logger,
+    label: 'clone history in project-history',
+    info: { targetProjectId, sourceProjectId },
+  })
+
+  WebApiManager.getHistoryId(targetProjectId, (err, targetHistoryId) => {
+    if (err) return incrResp.fail(OError.tag(err, 'get target historyId'))
+    WebApiManager.getHistoryId(sourceProjectId, (err, sourceHistoryId) => {
+      if (err) return incrResp.fail(OError.tag(err, 'get source historyId'))
+
+      incrResp.sendUpdate('cloning full project history data: pending')
+      HistoryStoreManager.cloneProject(
+        sourceHistoryId.toString(),
+        targetHistoryId.toString(),
+        incrResp.signal(),
+        (err, stream) => {
+          if (err) {
+            incrResp.fail(OError.tag(err, 'clone history-v1 data'))
+            return
+          }
+
+          // aborted. pipeline() would throw.
+          if (res.destroyed) {
+            stream.destroy()
+            incrResp.fail(new Error('request aborted'))
+            return
+          }
+
+          // The stream.pipeline callback API does not support options.
+          Stream.promises.pipeline(stream, res, { end: false }).then(
+            () => {
+              incrResp.sendUpdate('clone labels: pending')
+              LabelsManager.cloneLabels(
+                sourceProjectId,
+                targetProjectId,
+                err => {
+                  if (err) {
+                    incrResp.fail(OError.tag(err, 'clone labels'))
+                    return
+                  }
+                  incrResp.sendUpdate('clone labels: done')
+
+                  incrResp.sendUpdate('clone resync state: pending')
+                  SyncManager.cloneResyncState(
+                    sourceProjectId,
+                    targetProjectId,
+                    err => {
+                      if (err) {
+                        incrResp.fail(OError.tag(err, 'clone resync state'))
+                        return
+                      }
+                      incrResp.sendUpdate('clone resync state: done')
+
+                      incrResp.sendUpdate('clone failure record: pending')
+                      ErrorRecorder.cloneFailure(
+                        sourceProjectId,
+                        targetProjectId,
+                        err => {
+                          if (err) {
+                            incrResp.fail(OError.tag(err, 'clone failure'))
+                            return
+                          }
+                          incrResp.sendUpdate('clone failure record: done')
+
+                          incrResp.sendUpdate('done')
+                          incrResp.end()
+                        }
+                      )
+                    }
+                  )
+                }
+              )
+            },
+            err => {
+              incrResp.fail(OError.tag(err, 'stream history-v1 response'))
+            }
+          )
+        }
+      )
+    })
+  })
+}
+
 const getProjectBlobSchema = z.object({
   params: z.object({
     history_id: zz.objectId().or(z.coerce.number()),

services/project-history/app/js/LabelsManager.js

@@ -4,6 +4,26 @@ import * as HistoryStoreManager from './HistoryStoreManager.js'
 import * as UpdatesProcessor from './UpdatesProcessor.js'
 import * as WebApiManager from './WebApiManager.js'
 
+export function cloneLabels(sourceProjectId, targetProjectId, callback) {
+  db.projectHistoryLabels
+    .find({ project_id: new ObjectId(sourceProjectId) })
+    .project({ _id: 0, project_id: 0 })
+    .toArray((err, labels) => {
+      if (err) return callback(OError.tag(err))
+      if (labels.length === 0) return callback()
+      db.projectHistoryLabels.insertMany(
+        labels.map(label => ({
+          ...label,
+          project_id: new ObjectId(targetProjectId),
+        })),
+        err => {
+          if (err) return callback(OError.tag(err))
+          callback()
+        }
+      )
+    })
+}
+
 export function getLabels(projectId, callback) {
   _toObjectId(projectId, function (error, projectId) {
     if (error) {

services/project-history/app/js/Router.js

@@ -83,6 +83,8 @@ export function initialize(app) {
 
   app.get('/project/:history_id/blob/:hash', HttpController.getProjectBlob)
 
+  app.post('/project/:project_id/clone', HttpController.cloneProject)
+
   app.get('/status/failures', HttpController.getFailures)
 
   app.get('/status/queue', HttpController.getQueueCounts)

services/project-history/app/js/SyncManager.js

@@ -109,6 +109,23 @@ async function startResyncWithoutLock(projectId, options) {
   await setResyncState(projectId, syncState)
 }
 
+/**
+ * @param {string} sourceProjectId
+ * @param {string} targetProjectId
+ * @return {Promise<void>}
+ */
+async function cloneResyncState(sourceProjectId, targetProjectId) {
+  const rawSyncState = await db.projectHistorySyncState.findOne(
+    { project_id: new ObjectId(sourceProjectId) },
+    { projection: { _id: 0, project_id: 0 } }
+  )
+  if (!rawSyncState) return
+  await db.projectHistorySyncState.insertOne({
+    ...rawSyncState,
+    project_id: new ObjectId(targetProjectId),
+  })
+}
+
 /**
  * @param {string} projectId
  * @return {Promise<SyncState>}
@@ -1329,6 +1346,7 @@ function trackingDirectivesEqual(a, b) {
 
 // EXPORTS
 
+const cloneResyncStateCb = callbackify(cloneResyncState)
 const getResyncStateCb = callbackify(getResyncState)
 const startResyncCb = callbackify(startResync)
 const startResyncWithoutLockCb = callbackify(startResyncWithoutLock)
@@ -1373,6 +1391,7 @@ const expandSyncUpdatesCb = (
 }
 
 export {
+  cloneResyncStateCb as cloneResyncState,
   getResyncStateCb as getResyncState,
   startResyncCb as startResync,
   startResyncWithoutLockCb as startResyncWithoutLock,
@@ -1384,6 +1403,7 @@ export {
 }
 
 export const promises = {
+  cloneResyncState,
   getResyncState,
   startResync,
   startResyncWithoutLock,