[document-updater] adopt strict zod types and extend test coverage (#36077)

* [document-updater] adopt strict zod types and extend test coverage

* [saas-e2e] add test for project restore with binary and linked file

GitOrigin-RevId: a8e45bd1e46b6ac8a293da24c8b8ad7b236fd9bd
This commit is contained in:
Jakob Ackermann
2026-09-17 11:46:05 +00:00
committed by Copybot
parent a57d8449ef
commit 4e80098424
18 changed files with 1201 additions and 211 deletions
@@ -1,4 +1,7 @@
const { expressify } = require('@overleaf/promise-utils')
const { parseReq, z, zz } = require('@overleaf/validation-tools')
const editorCoreSchemas = require('overleaf-editor-core/lib/schemas')
const schemas = require('./schemas')
const DocumentManager = require('./DocumentManager')
const HistoryManager = require('./HistoryManager')
const ProjectManager = require('./ProjectManager')
@@ -12,20 +15,54 @@ const { getTotalSizeOfLines } = require('./Limits')
const { StringFileData } = require('overleaf-editor-core')
const HistoryConversions = require('./HistoryConversions')
const projectParamsSchema = z.strictObject({
project_id: zz.objectId(),
})
const docParamsSchema = z.strictObject({
project_id: zz.objectId(),
doc_id: zz.objectId(),
})
const commentParamsSchema = z.strictObject({
project_id: zz.objectId(),
doc_id: zz.objectId(),
comment_id: zz.objectId(),
})
const getDocSchema = z.object({
params: docParamsSchema,
query: z.strictObject({
historyOTSupport: z.stringbool().default(false),
fromVersion: z.coerce.number().int().default(-1),
}),
})
// Rollout-temporary fallback (loosened primary schema; no zod validation
// existed for this route on main); delete when this route's
// REQ_VALIDATION_MODE instrumentation is removed.
const getDocFallbackSchema = z.object({
params: z.object({
project_id: z.string(),
doc_id: z.string(),
}),
query: z.object({
historyOTSupport: z.stringbool().default(false),
fromVersion: z.coerce.number().int().default(-1),
}),
})
async function getDoc(req, res) {
let fromVersion
const docId = req.params.doc_id
const projectId = req.params.project_id
const { params, query } = parseReq(req, getDocSchema, {
logOnly: true,
fallbackSchema: getDocFallbackSchema,
})
const { project_id: projectId, doc_id: docId } = params
const { fromVersion } = query
logger.debug({ projectId, docId }, 'getting doc via http')
const timer = new Metrics.Timer('http.getDoc')
if (req.query.fromVersion != null) {
fromVersion = parseInt(req.query.fromVersion, 10)
} else {
fromVersion = -1
}
let { lines, version, ops, ranges, pathname, type } =
await DocumentManager.promises.getDocAndRecentOpsWithLock(
projectId,
@@ -39,7 +76,7 @@ async function getDoc(req, res) {
throw new Errors.NotFoundError('document not found')
}
if (!Array.isArray(lines) && req.query.historyOTSupport !== 'true') {
if (!Array.isArray(lines) && !query.historyOTSupport) {
;({ lines, ranges } = HistoryConversions.fromHistoryOT(lines))
}
@@ -55,10 +92,13 @@ async function getDoc(req, res) {
})
}
const getCommentSchema = z.object({
params: commentParamsSchema,
})
async function getComment(req, res) {
const docId = req.params.doc_id
const projectId = req.params.project_id
const commentId = req.params.comment_id
const { params } = parseReq(req, getCommentSchema, { logOnly: true })
const { project_id: projectId, doc_id: docId, comment_id: commentId } = params
logger.debug({ projectId, docId, commentId }, 'getting comment via http')
@@ -75,10 +115,33 @@ async function getComment(req, res) {
res.json(comment)
}
const peekDocSchema = z.object({
params: docParamsSchema,
query: z.strictObject({
historyOTSupport: z.stringbool().default(false),
}),
})
// Rollout-temporary fallback (loosened primary schema; no zod validation
// existed for this route on main); delete when this route's
// REQ_VALIDATION_MODE instrumentation is removed.
const peekDocFallbackSchema = z.object({
params: z.object({
project_id: z.string(),
doc_id: z.string(),
}),
query: z.object({
historyOTSupport: z.stringbool().default(false),
}),
})
// return the doc from redis if present, but don't load it from mongo
async function peekDoc(req, res) {
const docId = req.params.doc_id
const projectId = req.params.project_id
const { params, query } = parseReq(req, peekDocSchema, {
logOnly: true,
fallbackSchema: peekDocFallbackSchema,
})
const { project_id: projectId, doc_id: docId } = params
logger.debug({ projectId, docId }, 'peeking at doc via http')
let { lines, version } = await RedisManager.promises.getDoc(projectId, docId)
@@ -87,7 +150,7 @@ async function peekDoc(req, res) {
throw new Errors.NotFoundError('document not found')
}
if (!Array.isArray(lines) && req.query.historyOTSupport !== 'true') {
if (!Array.isArray(lines) && !query.historyOTSupport) {
const file = StringFileData.fromRaw(lines)
// TODO(24596): tc support for history-ot
lines = file.getLines()
@@ -96,32 +159,27 @@ async function peekDoc(req, res) {
res.json({ id: docId, lines, version })
}
const getProjectDocsAndFlushIfOldSchema = z.object({
params: projectParamsSchema,
query: z.strictObject({
state: z.string().optional(),
}),
})
async function getProjectDocsAndFlushIfOld(req, res) {
const projectId = req.params.project_id
const projectStateHash = req.query.state
// exclude is string of existing docs "id:version,id:version,..."
const excludeItems =
req.query.exclude != null ? req.query.exclude.split(',') : []
logger.debug({ projectId, exclude: excludeItems }, 'getting docs via http')
const { params, query } = parseReq(req, getProjectDocsAndFlushIfOldSchema, {
logOnly: true,
})
const projectId = params.project_id
const projectStateHash = query.state
logger.debug({ projectId }, 'getting docs via http')
const timer = new Metrics.Timer('http.getAllDocs')
const excludeVersions = {}
for (const item of excludeItems) {
const [id, version] = item.split(':')
excludeVersions[id] = version
}
logger.debug(
{ projectId, projectStateHash, excludeVersions },
'excluding versions'
)
let result
try {
result = await ProjectManager.promises.getProjectDocsAndFlushIfOld(
projectId,
projectStateHash,
excludeVersions
projectStateHash
)
} catch (error) {
if (error instanceof Errors.ProjectStateChangedError) {
@@ -142,8 +200,13 @@ async function getProjectDocsAndFlushIfOld(req, res) {
res.send(result)
}
const projectOnlySchema = z.object({
params: projectParamsSchema,
})
async function getProjectLastUpdatedAt(req, res) {
const projectId = req.params.project_id
const projectId = parseReq(req, projectOnlySchema, { logOnly: true }).params
.project_id
let timestamps =
await ProjectManager.promises.getProjectDocsTimestamps(projectId)
@@ -158,13 +221,15 @@ async function getProjectLastUpdatedAt(req, res) {
}
async function getProjectRanges(req, res) {
const projectId = req.params.project_id
const projectId = parseReq(req, projectOnlySchema, { logOnly: true }).params
.project_id
const docs = await ProjectManager.promises.getProjectRanges(projectId)
res.json({ docs })
}
async function clearProjectState(req, res) {
const projectId = req.params.project_id
const projectId = parseReq(req, projectOnlySchema, { logOnly: true }).params
.project_id
const timer = new Metrics.Timer('http.clearProjectState')
logger.debug({ projectId }, 'clearing project state via http')
await ProjectManager.promises.clearProjectState(projectId)
@@ -172,10 +237,21 @@ async function clearProjectState(req, res) {
res.sendStatus(200)
}
const setDocSchema = z.object({
params: docParamsSchema,
body: z.strictObject({
lines: z.array(z.string()),
source: z.string(),
user_id: zz.objectId().nullish(),
undoing: z.boolean().optional(),
trackChanges: z.boolean().optional(),
}),
})
async function setDoc(req, res) {
const docId = req.params.doc_id
const projectId = req.params.project_id
const { lines, source, user_id: userId, undoing, trackChanges } = req.body
const { params, body } = parseReq(req, setDocSchema, { logOnly: true })
const { project_id: projectId, doc_id: docId } = params
const { lines, source, user_id: userId, undoing, trackChanges } = body
if (trackChanges && !userId) {
return res.status(400).send('track changes requires a user id')
}
@@ -213,10 +289,22 @@ async function setDoc(req, res) {
res.json(result || {})
}
const appendToDocSchema = z.object({
params: docParamsSchema,
body: z.strictObject({
lines: z.array(z.string()),
source: z.string(),
user_id: zz.objectId().nullish(),
trackChanges: z.boolean().optional(),
}),
})
async function appendToDoc(req, res) {
const docId = req.params.doc_id
const projectId = req.params.project_id
const { lines, source, user_id: userId, trackChanges } = req.body
const { params, body } = parseReq(req, appendToDocSchema, {
logOnly: true,
})
const { project_id: projectId, doc_id: docId } = params
const { lines, source, user_id: userId, trackChanges } = body
if (trackChanges && !userId) {
return res.status(400).send('track changes requires a user id')
}
@@ -249,9 +337,13 @@ async function appendToDoc(req, res) {
res.json(result)
}
const docOnlySchema = z.object({
params: docParamsSchema,
})
async function flushDocIfLoaded(req, res) {
const docId = req.params.doc_id
const projectId = req.params.project_id
const { params } = parseReq(req, docOnlySchema, { logOnly: true })
const { project_id: projectId, doc_id: docId } = params
logger.debug({ projectId, docId }, 'flushing doc via http')
const timer = new Metrics.Timer('http.flushDoc')
await DocumentManager.promises.flushDocIfLoadedWithLock(projectId, docId)
@@ -260,10 +352,33 @@ async function flushDocIfLoaded(req, res) {
res.sendStatus(204) // No Content
}
const deleteDocSchema = z.object({
params: docParamsSchema,
query: z.strictObject({
ignore_flush_errors: z.stringbool().default(false),
}),
})
// Rollout-temporary fallback (loosened primary schema; no zod validation
// existed for this route on main); delete when this route's
// REQ_VALIDATION_MODE instrumentation is removed.
const deleteDocFallbackSchema = z.object({
params: z.object({
project_id: z.string(),
doc_id: z.string(),
}),
query: z.object({
ignore_flush_errors: z.stringbool().default(false),
}),
})
async function deleteDoc(req, res) {
const docId = req.params.doc_id
const projectId = req.params.project_id
const ignoreFlushErrors = req.query.ignore_flush_errors === 'true'
const { params, query } = parseReq(req, deleteDocSchema, {
logOnly: true,
fallbackSchema: deleteDocFallbackSchema,
})
const { project_id: projectId, doc_id: docId } = params
const ignoreFlushErrors = query.ignore_flush_errors
const timer = new Metrics.Timer('http.deleteDoc')
logger.debug({ projectId, docId }, 'deleting doc via http')
@@ -283,7 +398,8 @@ async function deleteDoc(req, res) {
}
async function flushProject(req, res) {
const projectId = req.params.project_id
const projectId = parseReq(req, projectOnlySchema, { logOnly: true }).params
.project_id
logger.debug({ projectId }, 'flushing project via http')
const timer = new Metrics.Timer('http.flushProject')
await ProjectManager.promises.flushProjectWithLocks(projectId)
@@ -292,17 +408,42 @@ async function flushProject(req, res) {
res.sendStatus(204) // No Content
}
const deleteProjectSchema = z.object({
params: projectParamsSchema,
query: z.strictObject({
background: z.stringbool().default(false),
shutdown: z.stringbool().default(false),
}),
})
// Rollout-temporary fallback (loosened primary schema; no zod validation
// existed for this route on main); delete when this route's
// REQ_VALIDATION_MODE instrumentation is removed.
const deleteProjectFallbackSchema = z.object({
params: z.object({
project_id: z.string(),
}),
query: z.object({
background: z.stringbool().default(false),
shutdown: z.stringbool().default(false),
}),
})
async function deleteProject(req, res) {
const projectId = req.params.project_id
const { params, query } = parseReq(req, deleteProjectSchema, {
logOnly: true,
fallbackSchema: deleteProjectFallbackSchema,
})
const projectId = params.project_id
logger.debug({ projectId }, 'deleting project via http')
const options = {}
if (req.query.background) {
if (query.background) {
options.background = true
} // allow non-urgent flushes to be queued
if (req.query.shutdown) {
if (query.shutdown) {
options.skip_history_flush = true
} // don't flush history when realtime shuts down
if (req.query.background) {
if (query.background) {
await ProjectManager.promises.queueFlushAndDeleteProject(projectId)
logger.debug({ projectId }, 'queue delete of project via http')
} else {
@@ -318,8 +459,26 @@ async function deleteProject(req, res) {
res.sendStatus(204)
}
const deleteMultipleProjectsSchema = z.object({
body: z.strictObject({
project_ids: z.array(zz.objectId()).default([]),
}),
})
// Rollout-temporary fallback (loosened primary schema; no zod validation
// existed for this route on main); delete when this route's
// REQ_VALIDATION_MODE instrumentation is removed.
const deleteMultipleProjectsFallbackSchema = z.object({
body: z.object({
project_ids: z.array(z.string()).default([]),
}),
})
async function deleteMultipleProjects(req, res) {
const projectIds = req.body.project_ids || []
const projectIds = parseReq(req, deleteMultipleProjectsSchema, {
logOnly: true,
fallbackSchema: deleteMultipleProjectsFallbackSchema,
}).body.project_ids
logger.debug({ projectIds }, 'deleting multiple projects via http')
for (const projectId of projectIds) {
logger.debug({ projectId }, 'queue delete of project via http')
@@ -328,11 +487,26 @@ async function deleteMultipleProjects(req, res) {
res.sendStatus(204) // No Content
}
// tracked-change ids are RangesTracker ids (seed + increment), not ObjectIds
const acceptChangesSchema = z.object({
params: z.strictObject({
project_id: zz.objectId(),
doc_id: zz.objectId(),
change_id: z.string().min(1).optional(),
}),
body: z.strictObject({
change_ids: z.array(z.string().min(1)).optional(),
}),
})
async function acceptChanges(req, res) {
const { project_id: projectId, doc_id: docId } = req.params
let changeIds = req.body.change_ids
const { params, body } = parseReq(req, acceptChangesSchema, {
logOnly: true,
})
const { project_id: projectId, doc_id: docId } = params
let changeIds = body.change_ids
if (changeIds == null) {
changeIds = [req.params.change_id]
changeIds = [params.change_id]
}
logger.debug(
{ projectId, docId },
@@ -354,13 +528,19 @@ async function acceptChanges(req, res) {
res.status(200).json({ changeContributors })
}
const commentWithUserSchema = z.object({
params: commentParamsSchema,
body: z.strictObject({
user_id: zz.objectId().optional(),
}),
})
async function resolveComment(req, res) {
const {
project_id: projectId,
doc_id: docId,
comment_id: commentId,
} = req.params
const userId = req.body.user_id
const { params, body } = parseReq(req, commentWithUserSchema, {
logOnly: true,
})
const { project_id: projectId, doc_id: docId, comment_id: commentId } = params
const userId = body.user_id
logger.debug({ projectId, docId, commentId }, 'resolving comment via http')
await DocumentManager.promises.updateCommentStateWithLock(
projectId,
@@ -374,12 +554,11 @@ async function resolveComment(req, res) {
}
async function reopenComment(req, res) {
const {
project_id: projectId,
doc_id: docId,
comment_id: commentId,
} = req.params
const userId = req.body.user_id
const { params, body } = parseReq(req, commentWithUserSchema, {
logOnly: true,
})
const { project_id: projectId, doc_id: docId, comment_id: commentId } = params
const userId = body.user_id
logger.debug({ projectId, docId, commentId }, 'reopening comment via http')
await DocumentManager.promises.updateCommentStateWithLock(
projectId,
@@ -393,12 +572,11 @@ async function reopenComment(req, res) {
}
async function deleteComment(req, res) {
const {
project_id: projectId,
doc_id: docId,
comment_id: commentId,
} = req.params
const userId = req.body.user_id
const { params, body } = parseReq(req, commentWithUserSchema, {
logOnly: true,
})
const { project_id: projectId, doc_id: docId, comment_id: commentId } = params
const userId = body.user_id
logger.debug({ projectId, docId, commentId }, 'deleting comment via http')
const timer = new Metrics.Timer('http.deleteComment')
await DocumentManager.promises.deleteCommentWithLock(
@@ -412,10 +590,114 @@ async function deleteComment(req, res) {
res.sendStatus(204) // No Content
}
// project-structure updates as built by web's DocumentUpdaterHandler
// (_getUpdates): deletes are renames to an empty newPathname
const renameUpdateSchema = z.strictObject({
type: z.enum(['rename-doc', 'rename-file']),
id: zz.objectId(),
// forwarded (via project-history) into history-v1's archive/zip builder
// (project_archive.js `archive.append(content, { name: pathname })`), so a
// traversal payload here is a zip-slip risk, not just a display string --
// see zod-string-field-hardening.md, category C.
pathname: zz.safePath(),
// deletes are renames to an empty newPathname (see the file comment
// above), so the empty string is a legitimate sentinel here, not a
// validation gap.
newPathname: zz.safePath().or(z.literal('')),
})
const addUpdateSchema = z.strictObject({
type: z.enum(['add-doc', 'add-file']),
id: zz.objectId(),
pathname: zz.safePath(),
docLines: z.string().optional(),
ranges: schemas.ranges.optional(),
historyRangesSupport: z.boolean().optional(),
// legacy filestore url for files without a created blob
url: z.string().nullish(),
hash: editorCoreSchemas.rawBlobHash.optional(),
metadata: editorCoreSchemas.rawFileMetadata.optional(),
createdBlob: z.boolean().optional(),
})
// Rollout-temporary fallback (loosened primary schema; no zod validation
// existed for this route on main); delete when this route's
// REQ_VALIDATION_MODE instrumentation is removed.
const renameUpdateFallbackSchema = z.object({
type: z.enum(['rename-doc', 'rename-file']),
id: z.string(),
pathname: z.string(),
newPathname: z.string(),
})
// Rollout-temporary fallback (loosened primary schema; no zod validation
// existed for this route on main); delete when this route's
// REQ_VALIDATION_MODE instrumentation is removed.
const addUpdateFallbackSchema = z.object({
type: z.enum(['add-doc', 'add-file']),
id: z.string(),
pathname: z.string(),
docLines: z.string().optional(),
ranges: z.object({}).passthrough().optional(),
historyRangesSupport: z.boolean().optional(),
url: z.string().nullish(),
hash: z.string().optional(),
metadata: z.object({}).passthrough().optional(),
createdBlob: z.boolean().optional(),
})
const updateProjectSchema = z.object({
params: projectParamsSchema,
body: z.strictObject({
projectHistoryId: z.union([z.number(), zz.projectHistoryId()]).optional(),
userId: zz.objectId().nullish(),
updates: z.array(renameUpdateSchema.or(addUpdateSchema)).default([]),
version: z.union([z.number(), z.string()]),
// `source` is polymorphic (see Utils.extractOriginOrSource): either a
// plain descriptive string (e.g. 'editor' for live user edits) or the
// richer Origin/RestoreOrigin/RestoreFileOrigin/RestoreProjectOrigin
// raw shape (see overleaf-editor-core/lib/origin/) that
// RestoreManager.mjs sends when reverting a file/project from history --
// it carries the version/timestamp of the restored version, which is
// needed to build a valid history Change origin downstream. web also
// sends an explicit `null` (not an omitted field) for system-initiated
// project-structure changes that aren't attributable to a specific
// editor action -- project creation (root/bib doc, template image),
// project duplication, and zip upload all call
// DocumentUpdaterHandler.updateProjectStructure with source=null.
source: editorCoreSchemas.rawOrigin.or(z.string()).nullish(),
}),
})
// Rollout-temporary fallback (loosened primary schema; no zod validation
// existed for this route on main); delete when this route's
// REQ_VALIDATION_MODE instrumentation is removed.
const updateProjectFallbackSchema = z.object({
params: z.object({
project_id: z.string(),
}),
body: z.object({
projectHistoryId: z.union([z.number(), z.string()]).optional(),
userId: z.string().nullish(),
updates: z
.array(renameUpdateFallbackSchema.or(addUpdateFallbackSchema))
.default([]),
version: z.union([z.number(), z.string()]),
// loosened equivalent of editorCoreSchemas.rawOrigin -- see the same
// simplification for `origin` in project-history's resyncProjectHistory
// fallback.
source: z.object({ kind: z.string() }).or(z.string()).nullish(),
}),
})
async function updateProject(req, res) {
const timer = new Metrics.Timer('http.updateProject')
const projectId = req.params.project_id
const { projectHistoryId, userId, updates = [], version, source } = req.body
const { params, body } = parseReq(req, updateProjectSchema, {
logOnly: true,
fallbackSchema: updateProjectFallbackSchema,
})
const projectId = params.project_id
const { projectHistoryId, userId, updates, version, source } = body
logger.debug({ projectId, updates, version }, 'updating project via http')
await ProjectManager.promises.updateProjectWithLocks(
projectId,
@@ -430,15 +712,44 @@ async function updateProject(req, res) {
res.sendStatus(204) // No Content
}
const resyncProjectHistorySchema = z.object({
params: projectParamsSchema,
body: z.strictObject({
projectHistoryId: z.union([z.number(), zz.projectHistoryId()]),
docs: z.array(
z.strictObject({
doc: zz.objectId(),
path: zz.safePath(),
})
),
files: z.array(
z.strictObject({
file: zz.objectId(),
path: zz.safePath(),
// legacy filestore url for files without a created blob
url: z.string().nullish(),
_hash: editorCoreSchemas.rawBlobHash.optional(),
createdBlob: z.boolean().optional(),
metadata: editorCoreSchemas.rawFileMetadata.optional(),
})
),
historyRangesMigration: z.string().optional(),
resyncProjectStructureOnly: z.boolean().optional(),
}),
})
async function resyncProjectHistory(req, res) {
const projectId = req.params.project_id
const { params, body } = parseReq(req, resyncProjectHistorySchema, {
logOnly: true,
})
const projectId = params.project_id
const {
projectHistoryId,
docs,
files,
historyRangesMigration,
resyncProjectStructureOnly,
} = req.body
} = body
logger.debug(
{ projectId, docs, files },
@@ -464,12 +775,39 @@ async function resyncProjectHistory(req, res) {
res.sendStatus(204)
}
const flushQueuedProjectsSchema = z.object({
query: z.strictObject({
limit: z.coerce.number().int().default(1000),
min_delete_age: z.coerce
.number()
.int()
.default(5 * 60 * 1000),
}),
})
// Rollout-temporary fallback (loosened primary schema; no zod validation
// existed for this route on main); delete when this route's
// REQ_VALIDATION_MODE instrumentation is removed.
const flushQueuedProjectsFallbackSchema = z.object({
query: z.object({
limit: z.coerce.number().int().default(1000),
min_delete_age: z.coerce
.number()
.int()
.default(5 * 60 * 1000),
}),
})
async function flushQueuedProjects(req, res) {
res.setTimeout(10 * 60 * 1000)
const { query } = parseReq(req, flushQueuedProjectsSchema, {
logOnly: true,
fallbackSchema: flushQueuedProjectsFallbackSchema,
})
const options = {
limit: req.query.limit || 1000,
limit: query.limit,
timeout: 5 * 60 * 1000,
min_delete_age: req.query.min_delete_age || 5 * 60 * 1000,
min_delete_age: query.min_delete_age,
}
await DeleteQueueManager.promises.flushAndDeleteOldProjects(
options,
@@ -492,7 +830,8 @@ async function flushQueuedProjects(req, res) {
* response indicates whether the project has been blocked or not.
*/
async function blockProject(req, res) {
const projectId = req.params.project_id
const projectId = parseReq(req, projectOnlySchema, { logOnly: true }).params
.project_id
const blocked = await RedisManager.promises.blockProject(projectId)
res.json({ blocked })
}
@@ -501,7 +840,8 @@ async function blockProject(req, res) {
* Unblock a project
*/
async function unblockProject(req, res) {
const projectId = req.params.project_id
const projectId = parseReq(req, projectOnlySchema, { logOnly: true }).params
.project_id
const wasBlocked = await RedisManager.promises.unblockProject(projectId)
res.json({ wasBlocked })
}