overleaf-cep/services/document-updater/app/js/HttpController.js

const { expressify } = require('@overleaf/promise-utils')
const DocumentManager = require('./DocumentManager')
const HistoryManager = require('./HistoryManager')
const ProjectManager = require('./ProjectManager')
const RedisManager = require('./RedisManager')
const Errors = require('./Errors')
const logger = require('@overleaf/logger')
const Settings = require('@overleaf/settings')
const Metrics = require('./Metrics')
const DeleteQueueManager = require('./DeleteQueueManager')
const { getTotalSizeOfLines } = require('./Limits')
const { StringFileData } = require('overleaf-editor-core')
const { addTrackedDeletesToContent } = require('./Utils')
const HistoryConversions = require('./HistoryConversions')

async function getDoc(req, res) {
  let fromVersion
  const docId = req.params.doc_id
  const projectId = req.params.project_id
  const historyRanges = req.query.historyRanges === 'true'

  logger.debug({ projectId, docId, historyRanges }, 'getting doc via http')
  const timer = new Metrics.Timer('http.getDoc')

  if (req.query.fromVersion != null) {
    fromVersion = parseInt(req.query.fromVersion, 10)
  } else {
    fromVersion = -1
  }

  let { lines, version, ops, ranges, pathname, type } =
    await DocumentManager.promises.getDocAndRecentOpsWithLock(
      projectId,
      docId,
      fromVersion
    )
  timer.done()
  logger.debug({ projectId, docId, historyRanges }, 'got doc via http')

  if (lines == null || version == null) {
    throw new Errors.NotFoundError('document not found')
  }

  if (!Array.isArray(lines) && req.query.historyOTSupport !== 'true') {
    const file = StringFileData.fromRaw(lines)
    // TODO(24596): tc support for history-ot
    lines = file.getLines()
  }

  if (historyRanges) {
    const docContentWithTrackedDeletes = addTrackedDeletesToContent(
      lines.join('\n'),
      ranges?.changes ?? []
    )
    const docLinesWithTrackedDeletes = docContentWithTrackedDeletes.split('\n')
    const rangesWithTrackedDeletes = HistoryConversions.toHistoryRanges(ranges)

    res.json({
      id: docId,
      lines: docLinesWithTrackedDeletes,
      version,
      ops,
      ranges: rangesWithTrackedDeletes,
      pathname,
      ttlInS: RedisManager.DOC_OPS_TTL,
      type,
    })
  } else {
    res.json({
      id: docId,
      lines,
      version,
      ops,
      ranges,
      pathname,
      ttlInS: RedisManager.DOC_OPS_TTL,
      type,
    })
  }
}

async function getComment(req, res) {
  const docId = req.params.doc_id
  const projectId = req.params.project_id
  const commentId = req.params.comment_id

  logger.debug({ projectId, docId, commentId }, 'getting comment via http')

  const comment = await DocumentManager.promises.getCommentWithLock(
    projectId,
    docId,
    commentId
  )

  if (comment == null) {
    throw new Errors.NotFoundError('comment not found')
  }

  res.json(comment)
}

// return the doc from redis if present, but don't load it from mongo
async function peekDoc(req, res) {
  const docId = req.params.doc_id
  const projectId = req.params.project_id

  logger.debug({ projectId, docId }, 'peeking at doc via http')
  let { lines, version } = await RedisManager.promises.getDoc(projectId, docId)

  if (lines == null || version == null) {
    throw new Errors.NotFoundError('document not found')
  }

  if (!Array.isArray(lines) && req.query.historyOTSupport !== 'true') {
    const file = StringFileData.fromRaw(lines)
    // TODO(24596): tc support for history-ot
    lines = file.getLines()
  }

  res.json({ id: docId, lines, version })
}

async function getProjectDocsAndFlushIfOld(req, res) {
  const projectId = req.params.project_id
  const projectStateHash = req.query.state
  // exclude is string of existing docs "id:version,id:version,..."
  const excludeItems =
    req.query.exclude != null ? req.query.exclude.split(',') : []
  logger.debug({ projectId, exclude: excludeItems }, 'getting docs via http')
  const timer = new Metrics.Timer('http.getAllDocs')
  const excludeVersions = {}

  for (const item of excludeItems) {
    const [id, version] = item.split(':')
    excludeVersions[id] = version
  }

  logger.debug(
    { projectId, projectStateHash, excludeVersions },
    'excluding versions'
  )

  let result
  try {
    result = await ProjectManager.promises.getProjectDocsAndFlushIfOld(
      projectId,
      projectStateHash,
      excludeVersions
    )
  } catch (error) {
    if (error instanceof Errors.ProjectStateChangedError) {
      return res.sendStatus(409) // conflict
    } else {
      throw error
    }
  }

  timer.done()
  logger.debug(
    {
      projectId,
      result: result.map(doc => `${doc._id}:${doc.v}`),
    },
    'got docs via http'
  )
  res.send(result)
}

async function getProjectLastUpdatedAt(req, res) {
  const projectId = req.params.project_id
  let timestamps =
    await ProjectManager.promises.getProjectDocsTimestamps(projectId)

  // Filter out nulls. This can happen when
  // - docs get flushed between the listing and getting the individual docs ts
  // - a doc flush failed half way (doc keys removed, project tracking not updated)
  timestamps = timestamps.filter(ts => !!ts)

  timestamps = timestamps.map(ts => parseInt(ts, 10))
  timestamps.sort((a, b) => (a > b ? 1 : -1))
  res.json({ lastUpdatedAt: timestamps.pop() })
}

async function getProjectRanges(req, res) {
  const projectId = req.params.project_id
  const docs = await ProjectManager.promises.getProjectRanges(projectId)
  res.json({ docs })
}

async function clearProjectState(req, res) {
  const projectId = req.params.project_id
  const timer = new Metrics.Timer('http.clearProjectState')
  logger.debug({ projectId }, 'clearing project state via http')
  await ProjectManager.promises.clearProjectState(projectId)
  timer.done()
  res.sendStatus(200)
}

async function setDoc(req, res) {
  const docId = req.params.doc_id
  const projectId = req.params.project_id
  const { lines, source, user_id: userId, undoing } = req.body
  const lineSize = getTotalSizeOfLines(lines)

  if (lineSize > Settings.max_doc_length) {
    logger.warn(
      { projectId, docId, source, lineSize, userId },
      'document too large, returning 406 response'
    )
    return res.sendStatus(406)
  }
  logger.debug(
    { projectId, docId, lines, source, userId, undoing },
    'setting doc via http'
  )
  const timer = new Metrics.Timer('http.setDoc')

  const result = await DocumentManager.promises.setDocWithLock(
    projectId,
    docId,
    lines,
    source,
    userId,
    undoing,
    true
  )
  timer.done()
  logger.debug({ projectId, docId }, 'set doc via http')

  // If the document is unchanged and hasn't been updated, `result` will be
  // undefined, which leads to an invalid JSON response, so we send an empty
  // object instead.
  res.json(result || {})
}

async function appendToDoc(req, res) {
  const docId = req.params.doc_id
  const projectId = req.params.project_id
  const { lines, source, user_id: userId } = req.body
  const timer = new Metrics.Timer('http.appendToDoc')

  let result
  try {
    result = await DocumentManager.promises.appendToDocWithLock(
      projectId,
      docId,
      lines,
      source,
      userId
    )
  } catch (error) {
    if (error instanceof Errors.FileTooLargeError) {
      logger.warn('refusing to append to file, it would become too large')
      return res.sendStatus(422)
    } else {
      throw error
    }
  }

  timer.done()
  logger.debug(
    { projectId, docId, lines, source, userId },
    'appending to doc via http'
  )
  res.json(result)
}

async function flushDocIfLoaded(req, res) {
  const docId = req.params.doc_id
  const projectId = req.params.project_id
  logger.debug({ projectId, docId }, 'flushing doc via http')
  const timer = new Metrics.Timer('http.flushDoc')
  await DocumentManager.promises.flushDocIfLoadedWithLock(projectId, docId)
  timer.done()
  logger.debug({ projectId, docId }, 'flushed doc via http')
  res.sendStatus(204) // No Content
}

async function deleteDoc(req, res) {
  const docId = req.params.doc_id
  const projectId = req.params.project_id
  const ignoreFlushErrors = req.query.ignore_flush_errors === 'true'
  const timer = new Metrics.Timer('http.deleteDoc')
  logger.debug({ projectId, docId }, 'deleting doc via http')

  try {
    await DocumentManager.promises.flushAndDeleteDocWithLock(projectId, docId, {
      ignoreFlushErrors,
    })
  } finally {
    timer.done()
    // There is no harm in flushing project history if the previous call
    // failed and sometimes it is required
    HistoryManager.flushProjectChangesAsync(projectId)
  }

  logger.debug({ projectId, docId }, 'deleted doc via http')
  res.sendStatus(204) // No Content
}

async function flushProject(req, res) {
  const projectId = req.params.project_id
  logger.debug({ projectId }, 'flushing project via http')
  const timer = new Metrics.Timer('http.flushProject')
  await ProjectManager.promises.flushProjectWithLocks(projectId)
  timer.done()
  logger.debug({ projectId }, 'flushed project via http')
  res.sendStatus(204) // No Content
}

async function deleteProject(req, res) {
  const projectId = req.params.project_id
  logger.debug({ projectId }, 'deleting project via http')
  const options = {}
  if (req.query.background) {
    options.background = true
  } // allow non-urgent flushes to be queued
  if (req.query.shutdown) {
    options.skip_history_flush = true
  } // don't flush history when realtime shuts down
  if (req.query.background) {
    await ProjectManager.promises.queueFlushAndDeleteProject(projectId)
    logger.debug({ projectId }, 'queue delete of project via http')
  } else {
    const timer = new Metrics.Timer('http.deleteProject')
    await ProjectManager.promises.flushAndDeleteProjectWithLocks(
      projectId,
      options
    )
    timer.done()
    logger.debug({ projectId }, 'deleted project via http')
  }

  res.sendStatus(204)
}

async function deleteMultipleProjects(req, res) {
  const projectIds = req.body.project_ids || []
  logger.debug({ projectIds }, 'deleting multiple projects via http')
  for (const projectId of projectIds) {
    logger.debug({ projectId }, 'queue delete of project via http')
    await ProjectManager.promises.queueFlushAndDeleteProject(projectId)
  }
  res.sendStatus(204) // No Content
}

async function acceptChanges(req, res) {
  const { project_id: projectId, doc_id: docId } = req.params
  let changeIds = req.body.change_ids
  if (changeIds == null) {
    changeIds = [req.params.change_id]
  }
  logger.debug(
    { projectId, docId },
    `accepting ${changeIds.length} changes via http`
  )
  const timer = new Metrics.Timer('http.acceptChanges')
  await DocumentManager.promises.acceptChangesWithLock(
    projectId,
    docId,
    changeIds
  )
  timer.done()
  logger.debug(
    { projectId, docId },
    `accepted ${changeIds.length} changes via http`
  )
  res.sendStatus(204) // No Content
}

async function rejectChanges(req, res) {
  const { project_id: projectId, doc_id: docId } = req.params
  const changeIds = req.body.change_ids
  const userId = req.body.user_id

  logger.debug(
    { projectId, docId },
    `rejecting ${changeIds.length} changes via http`
  )
  const response = await DocumentManager.promises.rejectChangesWithLock(
    projectId,
    docId,
    changeIds,
    userId
  )
  logger.debug(
    { projectId, docId, changeIds, response },
    `rejected ${changeIds.length} changes via http`
  )
  res.json(response)
}

async function resolveComment(req, res) {
  const {
    project_id: projectId,
    doc_id: docId,
    comment_id: commentId,
  } = req.params
  const userId = req.body.user_id
  logger.debug({ projectId, docId, commentId }, 'resolving comment via http')
  await DocumentManager.promises.updateCommentStateWithLock(
    projectId,
    docId,
    commentId,
    userId,
    true
  )
  logger.debug({ projectId, docId, commentId }, 'resolved comment via http')
  res.sendStatus(204) // No Content
}

async function reopenComment(req, res) {
  const {
    project_id: projectId,
    doc_id: docId,
    comment_id: commentId,
  } = req.params
  const userId = req.body.user_id
  logger.debug({ projectId, docId, commentId }, 'reopening comment via http')
  await DocumentManager.promises.updateCommentStateWithLock(
    projectId,
    docId,
    commentId,
    userId,
    false
  )
  logger.debug({ projectId, docId, commentId }, 'reopened comment via http')
  res.sendStatus(204) // No Content
}

async function deleteComment(req, res) {
  const {
    project_id: projectId,
    doc_id: docId,
    comment_id: commentId,
  } = req.params
  const userId = req.body.user_id
  logger.debug({ projectId, docId, commentId }, 'deleting comment via http')
  const timer = new Metrics.Timer('http.deleteComment')
  await DocumentManager.promises.deleteCommentWithLock(
    projectId,
    docId,
    commentId,
    userId
  )
  timer.done()
  logger.debug({ projectId, docId, commentId }, 'deleted comment via http')
  res.sendStatus(204) // No Content
}

async function updateProject(req, res) {
  const timer = new Metrics.Timer('http.updateProject')
  const projectId = req.params.project_id
  const { projectHistoryId, userId, updates = [], version, source } = req.body
  logger.debug({ projectId, updates, version }, 'updating project via http')
  await ProjectManager.promises.updateProjectWithLocks(
    projectId,
    projectHistoryId,
    userId,
    updates,
    version,
    source
  )
  timer.done()
  logger.debug({ projectId }, 'updated project via http')
  res.sendStatus(204) // No Content
}

async function resyncProjectHistory(req, res) {
  const projectId = req.params.project_id
  const {
    projectHistoryId,
    docs,
    files,
    historyRangesMigration,
    resyncProjectStructureOnly,
  } = req.body

  logger.debug(
    { projectId, docs, files },
    'queuing project history resync via http'
  )

  const opts = {}
  if (historyRangesMigration) {
    opts.historyRangesMigration = historyRangesMigration
  }
  if (resyncProjectStructureOnly) {
    opts.resyncProjectStructureOnly = resyncProjectStructureOnly
  }

  await HistoryManager.promises.resyncProjectHistory(
    projectId,
    projectHistoryId,
    docs,
    files,
    opts
  )
  logger.debug({ projectId }, 'queued project history resync via http')
  res.sendStatus(204)
}

async function flushQueuedProjects(req, res) {
  res.setTimeout(10 * 60 * 1000)
  const options = {
    limit: req.query.limit || 1000,
    timeout: 5 * 60 * 1000,
    min_delete_age: req.query.min_delete_age || 5 * 60 * 1000,
  }
  await DeleteQueueManager.promises.flushAndDeleteOldProjects(
    options,
    (err, flushed) => {
      if (err) {
        logger.err({ err }, 'error flushing old projects')
        res.sendStatus(500)
      } else {
        logger.info({ flushed }, 'flush of queued projects completed')
        res.send({ flushed })
      }
    }
  )
}

/**
 * Block a project from getting loaded in docupdater
 *
 * The project is blocked only if it's not already loaded in docupdater. The
 * response indicates whether the project has been blocked or not.
 */
async function blockProject(req, res) {
  const projectId = req.params.project_id
  const blocked = await RedisManager.promises.blockProject(projectId)
  res.json({ blocked })
}

/**
 * Unblock a project
 */
async function unblockProject(req, res) {
  const projectId = req.params.project_id
  const wasBlocked = await RedisManager.promises.unblockProject(projectId)
  res.json({ wasBlocked })
}

module.exports = {
  getDoc: expressify(getDoc),
  peekDoc: expressify(peekDoc),
  getProjectDocsAndFlushIfOld: expressify(getProjectDocsAndFlushIfOld),
  getProjectLastUpdatedAt: expressify(getProjectLastUpdatedAt),
  getProjectRanges: expressify(getProjectRanges),
  clearProjectState: expressify(clearProjectState),
  appendToDoc: expressify(appendToDoc),
  setDoc: expressify(setDoc),
  flushDocIfLoaded: expressify(flushDocIfLoaded),
  deleteDoc: expressify(deleteDoc),
  flushProject: expressify(flushProject),
  deleteProject: expressify(deleteProject),
  deleteMultipleProjects: expressify(deleteMultipleProjects),
  acceptChanges: expressify(acceptChanges),
  rejectChanges: expressify(rejectChanges),
  resolveComment: expressify(resolveComment),
  reopenComment: expressify(reopenComment),
  deleteComment: expressify(deleteComment),
  updateProject: expressify(updateProject),
  resyncProjectHistory: expressify(resyncProjectHistory),
  flushQueuedProjects: expressify(flushQueuedProjects),
  blockProject: expressify(blockProject),
  unblockProject: expressify(unblockProject),
  getComment: expressify(getComment),
}