mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-12 13:13:45 +01:00
[clsi] adopt strict zod types and extend test coverage (#36090)
GitOrigin-RevId: ce11933ccc10ac74baf5c78cec194b5532a357a2
This commit is contained in:
+8
-31
@@ -18,9 +18,9 @@ import express from 'express'
|
||||
|
||||
import net from 'node:net'
|
||||
import os from 'node:os'
|
||||
import OError from '@overleaf/o-error'
|
||||
import ConversionController from './app/js/ConversionController.js'
|
||||
import FileUploadMiddleware from './app/js/FileUploadMiddleware.js'
|
||||
import { handleValidationError } from '@overleaf/validation-tools'
|
||||
logger.initialize('clsi')
|
||||
logger.logger.serializers.clsiRequest = LoggerSerializers.clsiRequest
|
||||
|
||||
@@ -46,30 +46,6 @@ app.use(function (req, res, next) {
|
||||
next()
|
||||
})
|
||||
|
||||
app.param('project_id', function (req, res, next, projectId) {
|
||||
if (projectId?.match(/^[a-zA-Z0-9_-]+$/)) {
|
||||
next()
|
||||
} else {
|
||||
next(new Error('invalid project id'))
|
||||
}
|
||||
})
|
||||
|
||||
app.param('user_id', function (req, res, next, userId) {
|
||||
if (userId?.match(/^[0-9a-f]{24}$/)) {
|
||||
next()
|
||||
} else {
|
||||
next(new Error('invalid user id'))
|
||||
}
|
||||
})
|
||||
|
||||
app.param('build_id', function (req, res, next, buildId) {
|
||||
if (buildId?.match(OutputCacheManager.BUILD_REGEX)) {
|
||||
next()
|
||||
} else {
|
||||
next(new OError('invalid build id', { buildId }))
|
||||
}
|
||||
})
|
||||
|
||||
app.post(
|
||||
'/project/:project_id/compile',
|
||||
express.json({ limit: Settings.compileSizeLimit }),
|
||||
@@ -124,15 +100,14 @@ app.get(
|
||||
)
|
||||
|
||||
// Conversion endpoints
|
||||
// Keep old route for backwards compatibility during CLSI/web deploy transition
|
||||
// Keep old route for backwards compatibility during CLSI/web deploy transition.
|
||||
// conversionType is threaded in directly (rather than via a query.type-setting
|
||||
// middleware) so the shared handler never needs to distinguish "the client
|
||||
// sent type=docx" from "this route only ever means docx".
|
||||
app.post(
|
||||
'/convert/docx-to-latex',
|
||||
FileUploadMiddleware.multerMiddleware,
|
||||
(req, res, next) => {
|
||||
req.query.type = 'docx'
|
||||
next()
|
||||
},
|
||||
ConversionController.convertDocumentToLaTeX
|
||||
ConversionController.convertDocxToLaTeX
|
||||
)
|
||||
app.post(
|
||||
'/convert/document-to-latex',
|
||||
@@ -216,6 +191,8 @@ app.get(
|
||||
async (req, res, next) => await smokeTest.sendNewResult(res).catch(next)
|
||||
)
|
||||
|
||||
app.use(handleValidationError)
|
||||
|
||||
app.use(function (error, req, res, next) {
|
||||
if (error instanceof Errors.NotFoundError) {
|
||||
logger.debug({ err: error, url: req.url }, 'not found error')
|
||||
|
||||
@@ -8,6 +8,8 @@ import ProjectPersistenceManager from './ProjectPersistenceManager.js'
|
||||
import logger from '@overleaf/logger'
|
||||
import Errors from './Errors.js'
|
||||
import CLSICacheHandler from './CLSICacheHandler.js'
|
||||
import { parseReq, z, zz } from '@overleaf/validation-tools'
|
||||
import { compileRequestBodySchema } from './schemas.js'
|
||||
|
||||
const { notifyCLSICacheAboutBuild } = CLSICacheHandler
|
||||
|
||||
@@ -17,16 +19,43 @@ function timeSinceLastSuccessfulCompile() {
|
||||
return Date.now() - lastSuccessfulCompileTimestamp
|
||||
}
|
||||
|
||||
// project_id is a Mongo ObjectId in the common case, but this service is also
|
||||
// hit directly by clsi-perf and by v1 submission/export flows using a bare
|
||||
// alphanumeric submission id (see zz.objectId().or(zz.submissionId()) in
|
||||
// web's ClsiURLHelpers.mjs, which this mirrors); user_id, when present, is
|
||||
// always a Mongo ObjectId.
|
||||
const projectOrUserParamsSchema = z.strictObject({
|
||||
project_id: zz.objectId().or(zz.submissionId()),
|
||||
user_id: zz.objectId().optional(),
|
||||
})
|
||||
|
||||
// web's ClsiManager.mjs (_getCompilerUrl / _makeRequestWithClsiServerId)
|
||||
// appends these to every GET it makes against this service, for backend
|
||||
// routing and metrics; none of them are read by the handlers below. Spread
|
||||
// into each affected route's strictObject query schema so genuine requests
|
||||
// aren't rejected.
|
||||
const clsiRoutingQueryFields = {
|
||||
compileBackendClass: zz.compileBackendClass().optional(),
|
||||
compileGroup: zz.compileGroup().optional(),
|
||||
clsiserverid: zz.clsiServerId().optional(),
|
||||
}
|
||||
|
||||
const compileSchema = z.object({
|
||||
params: projectOrUserParamsSchema,
|
||||
body: compileRequestBodySchema,
|
||||
})
|
||||
|
||||
function compile(req, res, next) {
|
||||
const { params, body } = parseReq(req, compileSchema, { logOnly: true })
|
||||
const timer = new Metrics.Timer('compile-request')
|
||||
RequestParser.parse(req.body, function (error, request) {
|
||||
RequestParser.parse(body, function (error, request) {
|
||||
if (error) {
|
||||
return next(error)
|
||||
}
|
||||
timer.opts = request.metricsOpts
|
||||
request.project_id = req.params.project_id
|
||||
if (req.params.user_id != null) {
|
||||
request.user_id = req.params.user_id
|
||||
request.project_id = params.project_id
|
||||
if (params.user_id != null) {
|
||||
request.user_id = params.user_id
|
||||
}
|
||||
ProjectPersistenceManager.markProjectAsJustAccessed(
|
||||
request.project_id,
|
||||
@@ -180,8 +209,15 @@ function compile(req, res, next) {
|
||||
})
|
||||
}
|
||||
|
||||
const projectOrUserOnlyParamsSchema = z.object({
|
||||
params: projectOrUserParamsSchema,
|
||||
})
|
||||
|
||||
function stopCompile(req, res, next) {
|
||||
const { project_id: projectId, user_id: userId } = req.params
|
||||
const { params } = parseReq(req, projectOrUserOnlyParamsSchema, {
|
||||
logOnly: true,
|
||||
})
|
||||
const { project_id: projectId, user_id: userId } = params
|
||||
CompileManager.stopCompile(projectId, userId, function (error) {
|
||||
if (error) {
|
||||
return next(error)
|
||||
@@ -191,7 +227,10 @@ function stopCompile(req, res, next) {
|
||||
}
|
||||
|
||||
function clearCache(req, res, next) {
|
||||
const { project_id: projectId, user_id: userId } = req.params
|
||||
const { params } = parseReq(req, projectOrUserOnlyParamsSchema, {
|
||||
logOnly: true,
|
||||
})
|
||||
const { project_id: projectId, user_id: userId } = params
|
||||
CompileManager.stopCompile(projectId, userId, error => {
|
||||
if (error) return next(OError.tag(error, 'stop compile'))
|
||||
ProjectPersistenceManager.clearProject(projectId, userId, error => {
|
||||
@@ -201,14 +240,51 @@ function clearCache(req, res, next) {
|
||||
})
|
||||
}
|
||||
|
||||
// imageName is validated against the allowlist enforced by
|
||||
// CompileManager._runSynctex (Errors.InvalidParameter -> 400), same as
|
||||
// before, since (like compile.options.imageName) it has no fixed vocabulary
|
||||
// independent of deployment settings.
|
||||
const syncFromCodeSchema = z.object({
|
||||
params: projectOrUserParamsSchema,
|
||||
query: z.strictObject({
|
||||
file: zz.filepath(),
|
||||
line: z.coerce.number().int(),
|
||||
column: z.coerce.number().int(),
|
||||
imageName: z.string().optional(),
|
||||
editorId: z.uuid().optional(),
|
||||
buildId: zz.buildId().optional(),
|
||||
compileFromClsiCache: z.stringbool().default(false),
|
||||
...clsiRoutingQueryFields,
|
||||
}),
|
||||
})
|
||||
|
||||
// Rollout-temporary fallback (loosened primary schema; no zod validation
|
||||
// existed for this route on main); delete when this route's
|
||||
// REQ_VALIDATION_MODE instrumentation is removed.
|
||||
const syncFromCodeFallbackSchema = z.object({
|
||||
params: z.object({
|
||||
project_id: z.string(),
|
||||
user_id: z.string().optional(),
|
||||
}),
|
||||
query: z.object({
|
||||
file: z.string(),
|
||||
line: z.coerce.number(),
|
||||
column: z.coerce.number(),
|
||||
imageName: z.string().optional(),
|
||||
editorId: z.string().optional(),
|
||||
buildId: z.string().optional(),
|
||||
compileFromClsiCache: z.stringbool().default(false),
|
||||
}),
|
||||
})
|
||||
|
||||
function syncFromCode(req, res, next) {
|
||||
const { file, editorId, buildId } = req.query
|
||||
const compileFromClsiCache = req.query.compileFromClsiCache === 'true'
|
||||
const line = parseInt(req.query.line, 10)
|
||||
const column = parseInt(req.query.column, 10)
|
||||
const { imageName } = req.query
|
||||
const projectId = req.params.project_id
|
||||
const userId = req.params.user_id
|
||||
const { params, query } = parseReq(req, syncFromCodeSchema, {
|
||||
logOnly: true,
|
||||
fallbackSchema: syncFromCodeFallbackSchema,
|
||||
})
|
||||
const { file, editorId, buildId, compileFromClsiCache, line, column } = query
|
||||
const { imageName } = query
|
||||
const { project_id: projectId, user_id: userId } = params
|
||||
CompileManager.syncFromCode(
|
||||
projectId,
|
||||
userId,
|
||||
@@ -228,14 +304,47 @@ function syncFromCode(req, res, next) {
|
||||
)
|
||||
}
|
||||
|
||||
const syncFromPdfSchema = z.object({
|
||||
params: projectOrUserParamsSchema,
|
||||
query: z.strictObject({
|
||||
page: z.coerce.number().int(),
|
||||
h: z.coerce.number(),
|
||||
v: z.coerce.number(),
|
||||
imageName: z.string().optional(),
|
||||
editorId: z.uuid().optional(),
|
||||
buildId: zz.buildId().optional(),
|
||||
compileFromClsiCache: z.stringbool().default(false),
|
||||
...clsiRoutingQueryFields,
|
||||
}),
|
||||
})
|
||||
|
||||
// Rollout-temporary fallback (loosened primary schema; no zod validation
|
||||
// existed for this route on main); delete when this route's
|
||||
// REQ_VALIDATION_MODE instrumentation is removed.
|
||||
const syncFromPdfFallbackSchema = z.object({
|
||||
params: z.object({
|
||||
project_id: z.string(),
|
||||
user_id: z.string().optional(),
|
||||
}),
|
||||
query: z.object({
|
||||
page: z.coerce.number(),
|
||||
h: z.coerce.number(),
|
||||
v: z.coerce.number(),
|
||||
imageName: z.string().optional(),
|
||||
editorId: z.string().optional(),
|
||||
buildId: z.string().optional(),
|
||||
compileFromClsiCache: z.stringbool().default(false),
|
||||
}),
|
||||
})
|
||||
|
||||
function syncFromPdf(req, res, next) {
|
||||
const page = parseInt(req.query.page, 10)
|
||||
const h = parseFloat(req.query.h)
|
||||
const v = parseFloat(req.query.v)
|
||||
const { imageName, editorId, buildId } = req.query
|
||||
const compileFromClsiCache = req.query.compileFromClsiCache === 'true'
|
||||
const projectId = req.params.project_id
|
||||
const userId = req.params.user_id
|
||||
const { params, query } = parseReq(req, syncFromPdfSchema, {
|
||||
logOnly: true,
|
||||
fallbackSchema: syncFromPdfFallbackSchema,
|
||||
})
|
||||
const { page, h, v, editorId, buildId, compileFromClsiCache } = query
|
||||
const { imageName } = query
|
||||
const { project_id: projectId, user_id: userId } = params
|
||||
CompileManager.syncFromPdf(
|
||||
projectId,
|
||||
userId,
|
||||
@@ -255,11 +364,36 @@ function syncFromPdf(req, res, next) {
|
||||
)
|
||||
}
|
||||
|
||||
const wordcountSchema = z.object({
|
||||
params: projectOrUserParamsSchema,
|
||||
query: z.strictObject({
|
||||
file: zz.filepath().default('main.tex'),
|
||||
image: z.string().optional(),
|
||||
...clsiRoutingQueryFields,
|
||||
}),
|
||||
})
|
||||
|
||||
// Rollout-temporary fallback (loosened primary schema; no zod validation
|
||||
// existed for this route on main); delete when this route's
|
||||
// REQ_VALIDATION_MODE instrumentation is removed.
|
||||
const wordcountFallbackSchema = z.object({
|
||||
params: z.object({
|
||||
project_id: z.string(),
|
||||
user_id: z.string().optional(),
|
||||
}),
|
||||
query: z.object({
|
||||
file: z.string().default('main.tex'),
|
||||
image: z.string().optional(),
|
||||
}),
|
||||
})
|
||||
|
||||
function wordcount(req, res, next) {
|
||||
const file = req.query.file || 'main.tex'
|
||||
const projectId = req.params.project_id
|
||||
const userId = req.params.user_id
|
||||
const { image } = req.query
|
||||
const { params, query } = parseReq(req, wordcountSchema, {
|
||||
logOnly: true,
|
||||
fallbackSchema: wordcountFallbackSchema,
|
||||
})
|
||||
const { file, image } = query
|
||||
const { project_id: projectId, user_id: userId } = params
|
||||
logger.debug({ image, file, projectId }, 'word count request')
|
||||
|
||||
CompileManager.wordcount(
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
import Path from 'node:path'
|
||||
import send from 'send'
|
||||
import Settings from '@overleaf/settings'
|
||||
import OutputCacheManager from './OutputCacheManager.js'
|
||||
|
||||
const ONE_DAY_S = 24 * 60 * 60
|
||||
const ONE_DAY_MS = ONE_DAY_S * 1000
|
||||
|
||||
function getPdfRange(req, res, next) {
|
||||
const { projectId, userId, contentId, hash } = req.params
|
||||
const perUserDir = userId ? `${projectId}-${userId}` : projectId
|
||||
const path = Path.join(
|
||||
Settings.path.outputDir,
|
||||
perUserDir,
|
||||
OutputCacheManager.CONTENT_SUBDIR,
|
||||
contentId,
|
||||
hash
|
||||
)
|
||||
res.setHeader('cache-control', `public, max-age=${ONE_DAY_S}`)
|
||||
res.setHeader('expires', new Date(Date.now() + ONE_DAY_MS).toUTCString())
|
||||
send(req, path).pipe(res)
|
||||
}
|
||||
|
||||
export default { getPdfRange }
|
||||
@@ -14,7 +14,8 @@ import RequestParser from './RequestParser.js'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import Settings from '@overleaf/settings'
|
||||
import Path from 'node:path'
|
||||
import { z } from '@overleaf/validation-tools'
|
||||
import { parseReq, z, zz } from '@overleaf/validation-tools'
|
||||
import { compileRequestBodySchema } from './schemas.js'
|
||||
|
||||
const CONVERSION_CONFIGS = {
|
||||
docx: { extension: 'docx' },
|
||||
@@ -22,17 +23,20 @@ const CONVERSION_CONFIGS = {
|
||||
html: { extension: 'zip' },
|
||||
}
|
||||
|
||||
async function convertDocumentToLaTeX(req, res) {
|
||||
const { path } = req.file
|
||||
const conversionType = req.query.type
|
||||
if (!Settings.enablePandocConversions) {
|
||||
await fs.unlink(path).catch(() => {})
|
||||
return res.sendStatus(404)
|
||||
}
|
||||
if (!conversionType || !['docx', 'markdown'].includes(conversionType)) {
|
||||
await fs.unlink(path).catch(() => {})
|
||||
return res.sendStatus(400)
|
||||
// Shared by every multer route: validate the schema and, if that fails,
|
||||
// still clean up the uploaded temp file before propagating the error (the
|
||||
// upload is guaranteed present by FileUploadMiddleware, so it is safe to
|
||||
// unlink even though the schema hasn't validated it yet).
|
||||
async function parseUploadedFileReq(req, schema, opts) {
|
||||
try {
|
||||
return parseReq(req, schema, opts)
|
||||
} catch (err) {
|
||||
await fs.unlink(req.file.path).catch(() => {})
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
async function runDocumentToLaTeXConversion(res, path, conversionType) {
|
||||
logger.debug({ path, conversionType }, 'received file for conversion')
|
||||
const conversionId = crypto.randomUUID()
|
||||
let zipPath
|
||||
@@ -79,22 +83,72 @@ async function convertDocumentToLaTeX(req, res) {
|
||||
}
|
||||
}
|
||||
|
||||
const PDFToJPEGQuerySchema = z.object({
|
||||
mode: z.enum(['preview', 'thumbnail']),
|
||||
const uploadedFileOnlySchema = z.object({ file: zz.uploadedFile() })
|
||||
|
||||
const convertDocumentToLaTeXSchema = z.object({
|
||||
// web's DocumentConversionManager.mjs always appends compileBackendClass
|
||||
// and compileGroup to this route's URL (same as CompileController.js's
|
||||
// clsiRoutingQueryFields), so the strict query schema must allow them too.
|
||||
query: z.strictObject({
|
||||
type: z.enum(['docx', 'markdown']),
|
||||
compileBackendClass: zz.compileBackendClass().optional(),
|
||||
compileGroup: zz.compileGroup().optional(),
|
||||
}),
|
||||
file: zz.uploadedFile(),
|
||||
})
|
||||
|
||||
async function convertDocumentToLaTeX(req, res) {
|
||||
if (!Settings.enablePandocConversions) {
|
||||
await fs.unlink(req.file.path).catch(() => {})
|
||||
return res.sendStatus(404)
|
||||
}
|
||||
const { file, query } = await parseUploadedFileReq(
|
||||
req,
|
||||
convertDocumentToLaTeXSchema,
|
||||
{ logOnly: true }
|
||||
)
|
||||
await runDocumentToLaTeXConversion(res, file.path, query.type)
|
||||
}
|
||||
|
||||
// Legacy alias of convertDocumentToLaTeX, kept for backwards compatibility
|
||||
// during CLSI/web deploy transitions (see app.js): conversionType is fixed to
|
||||
// 'docx' rather than read from the query string.
|
||||
async function convertDocxToLaTeX(req, res) {
|
||||
if (!Settings.enablePandocConversions) {
|
||||
await fs.unlink(req.file.path).catch(() => {})
|
||||
return res.sendStatus(404)
|
||||
}
|
||||
const { file } = await parseUploadedFileReq(req, uploadedFileOnlySchema, {
|
||||
logOnly: true,
|
||||
})
|
||||
await runDocumentToLaTeXConversion(res, file.path, 'docx')
|
||||
}
|
||||
|
||||
const convertPDFToJPEGSchema = z.object({
|
||||
query: z.strictObject({
|
||||
mode: z.enum(['preview', 'thumbnail']),
|
||||
// v1's CLSI::PdfToJpeg always appends this (Rails.application.config
|
||||
// .conversion_compile_backend_class); clsi-lb's haproxy config already
|
||||
// used it to route the request to the right backend pool
|
||||
// (url_param(compileBackendClass), see clsi-lb/app/js/haproxy-config.ts),
|
||||
// so the handler below doesn't need to read it again.
|
||||
compileBackendClass: zz.compileBackendClass().optional(),
|
||||
}),
|
||||
file: zz.uploadedFile(),
|
||||
})
|
||||
|
||||
async function convertPDFToJPEG(req, res) {
|
||||
const { path } = req.file
|
||||
if (!Settings.enablePdfConversions) {
|
||||
await fs.unlink(path).catch(() => {})
|
||||
await fs.unlink(req.file.path).catch(() => {})
|
||||
return res.sendStatus(404)
|
||||
}
|
||||
const parsed = PDFToJPEGQuerySchema.safeParse(req.query)
|
||||
if (!parsed.success) {
|
||||
await fs.unlink(path).catch(() => {})
|
||||
return res.sendStatus(400)
|
||||
}
|
||||
const { mode } = parsed.data
|
||||
const { file, query } = await parseUploadedFileReq(
|
||||
req,
|
||||
convertPDFToJPEGSchema,
|
||||
{ logOnly: true }
|
||||
)
|
||||
const { path } = file
|
||||
const { mode } = query
|
||||
logger.debug({ path, mode }, 'received pdf for conversion to jpeg')
|
||||
const conversionId = crypto.randomUUID()
|
||||
let jpegPath
|
||||
@@ -124,19 +178,61 @@ async function convertPDFToJPEG(req, res) {
|
||||
}
|
||||
}
|
||||
|
||||
// type/CONVERSION_CONFIGS keys kept in sync explicitly (rather than deriving
|
||||
// the enum from Object.keys) so the valid values are visible at a glance.
|
||||
const convertProjectToDocumentSchema = z.object({
|
||||
params: z.strictObject({
|
||||
project_id: zz.objectId().or(zz.submissionId()),
|
||||
user_id: zz.objectId(),
|
||||
}),
|
||||
// web's DocumentConversionManager.mjs always appends compileBackendClass
|
||||
// and compileGroup to this route's URL (same as CompileController.js's
|
||||
// clsiRoutingQueryFields), so the strict query schema must allow them too.
|
||||
query: z.strictObject({
|
||||
type: z.enum(['docx', 'markdown', 'html']),
|
||||
responseFormat: z.enum(['json', 'stream']).optional().default('stream'),
|
||||
compileBackendClass: zz.compileBackendClass().optional(),
|
||||
compileGroup: zz.compileGroup().optional(),
|
||||
}),
|
||||
body: compileRequestBodySchema,
|
||||
})
|
||||
|
||||
// Rollout-temporary fallback (loosened primary schema; no zod validation
|
||||
// existed for this route on main); delete when this route's
|
||||
// REQ_VALIDATION_MODE instrumentation is removed. `body` is reused as-is:
|
||||
// compileRequestBodySchema has no coercions/defaults of its own, so a raw
|
||||
// passthrough of it on failure hands the handler the same shape it always
|
||||
// has (RequestParser does its own, non-zod parsing of the body).
|
||||
const convertProjectToDocumentFallbackSchema = z.object({
|
||||
params: z.object({
|
||||
project_id: z.string(),
|
||||
user_id: z.string(),
|
||||
}),
|
||||
query: z.object({
|
||||
type: z.enum(['docx', 'markdown', 'html']),
|
||||
responseFormat: z.enum(['json', 'stream']).optional().default('stream'),
|
||||
}),
|
||||
body: compileRequestBodySchema,
|
||||
})
|
||||
|
||||
async function convertProjectToDocument(req, res) {
|
||||
if (!Settings.enablePandocConversions) {
|
||||
return res.sendStatus(404)
|
||||
}
|
||||
|
||||
const { user_id: userId, project_id: projectId } = req.params
|
||||
const type = req.query.type
|
||||
if (!Object.hasOwn(CONVERSION_CONFIGS, type)) {
|
||||
return res.sendStatus(400)
|
||||
}
|
||||
const { params, query, body } = parseReq(
|
||||
req,
|
||||
convertProjectToDocumentSchema,
|
||||
{
|
||||
logOnly: true,
|
||||
fallbackSchema: convertProjectToDocumentFallbackSchema,
|
||||
}
|
||||
)
|
||||
const { project_id: projectId, user_id: userId } = params
|
||||
const { type, responseFormat } = query
|
||||
const config = CONVERSION_CONFIGS[type]
|
||||
|
||||
const request = await RequestParser.promises.parse(req.body)
|
||||
const request = await RequestParser.promises.parse(body)
|
||||
request.project_id = projectId
|
||||
request.user_id = userId
|
||||
request.metricsOpts = {}
|
||||
@@ -144,8 +240,6 @@ async function convertProjectToDocument(req, res) {
|
||||
// the converted PDF bytes would be fed to pandoc instead of the original png.
|
||||
request.png2pdf = false
|
||||
|
||||
const responseFormat = req.query.responseFormat === 'json' ? 'json' : 'stream'
|
||||
|
||||
const conversionId = crypto.randomUUID()
|
||||
const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
|
||||
const conversionCacheDir = Path.join(Settings.path.clsiCacheDir, conversionId)
|
||||
@@ -257,6 +351,7 @@ async function convertProjectToDocument(req, res) {
|
||||
|
||||
export default {
|
||||
convertDocumentToLaTeX: expressify(convertDocumentToLaTeX),
|
||||
convertDocxToLaTeX: expressify(convertDocxToLaTeX),
|
||||
convertProjectToDocument: expressify(convertProjectToDocument),
|
||||
convertPDFToJPEG: expressify(convertPDFToJPEG),
|
||||
}
|
||||
|
||||
@@ -1,13 +1,21 @@
|
||||
import OutputFileArchiveManager from './OutputFileArchiveManager.js'
|
||||
import { expressify } from '@overleaf/promise-utils'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import { parseReq, z, zz } from '@overleaf/validation-tools'
|
||||
|
||||
// user_id is present only on the /project/:project_id/user/:user_id/build/...
|
||||
// mount; this same handler is used for both routes.
|
||||
const createOutputZipSchema = z.object({
|
||||
params: z.strictObject({
|
||||
project_id: zz.objectId().or(zz.submissionId()),
|
||||
user_id: zz.objectId().optional(),
|
||||
build_id: zz.buildId(),
|
||||
}),
|
||||
})
|
||||
|
||||
async function createOutputZip(req, res) {
|
||||
const {
|
||||
project_id: projectId,
|
||||
user_id: userId,
|
||||
build_id: buildId,
|
||||
} = req.params
|
||||
const { params } = parseReq(req, createOutputZipSchema, { logOnly: true })
|
||||
const { project_id: projectId, user_id: userId, build_id: buildId } = params
|
||||
|
||||
const archive = await OutputFileArchiveManager.archiveFilesForBuild(
|
||||
projectId,
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
// @ts-check
|
||||
// Zod schemas for the CLSI compile request body (the `{ compile: {...} }`
|
||||
// payload posted to /project/:project_id/compile and to
|
||||
// /project/:project_id/user/:user_id/download/project-to-document, which
|
||||
// re-parses the same payload via RequestParser for document conversions).
|
||||
//
|
||||
// RequestParser.parse() (see ./RequestParser.js) already performs the deep,
|
||||
// well-tested semantic validation of this payload: enum/valid-value checks
|
||||
// (compiler, imageName against the configured allowlist, syncType),
|
||||
// regex checks (historyId, editorId, buildId), timeout clamping, and the
|
||||
// rootResourcePath/filestoreBlobPrefix traversal check. Its behaviour
|
||||
// (including the specific error messages and the fact that a validation
|
||||
// failure currently surfaces as a 500, e.g. AllowedImageNamesTests.js) is
|
||||
// relied upon by both unit and acceptance tests, so this schema deliberately
|
||||
// only models the wire *shape* (fields and their JS/JSON types) and leaves
|
||||
// that semantic validation in place downstream, unchanged. The exception is
|
||||
// rootResourcePath, which is also given the standard zz.filepath() traversal
|
||||
// check here: it is a genuine path-traversal risk, the tightening is
|
||||
// consistent with RequestParser's own _checkPath, and no test relies on the
|
||||
// previous (500-producing) behaviour for a bad rootResourcePath.
|
||||
//
|
||||
// `token` (present in an old RequestParser unit-test fixture but never read
|
||||
// by RequestParser, CompileManager or any known caller in this repo) is
|
||||
// intentionally not part of the schema: every request field is expected to
|
||||
// be modeled, and nothing sends or reads this one.
|
||||
import { z, zz } from '@overleaf/validation-tools'
|
||||
import editorCoreSchemas from 'overleaf-editor-core/lib/schemas.js'
|
||||
|
||||
// Kept in sync explicitly with RequestParser's VALID_COMPILERS: importing it
|
||||
// from there isn't viable, as many unit tests mock the RequestParser module
|
||||
// down to an empty object, and this schema module is loaded transitively
|
||||
// through those mocks.
|
||||
const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex']
|
||||
|
||||
// Kept in sync explicitly with services/clsi-perf/app/js/Variants.js's
|
||||
// setup(), which derives one variant per services/clsi-perf/app/js/variants/
|
||||
// directory (currently example-project-frog, jpeg-images, memoir-manual,
|
||||
// minimal), plus a `${name}-gvisor` variant for the special-cased "minimal"
|
||||
// entry. HistoryResourceWriter.js's BlobStore#getBlobURL interpolates this
|
||||
// straight into a URL pathname (`u.pathname =
|
||||
// \`/variant/${clsiPerfVariant}/hash/${hash}\``), which -- unlike a template
|
||||
// literal appended to an existing path -- re-resolves ".." segments against
|
||||
// it, so an arbitrary string here is a real route/path dispatch confusion
|
||||
// risk, not just a cosmetic one.
|
||||
const VALID_CLSI_PERF_VARIANTS = [
|
||||
'example-project-frog',
|
||||
'jpeg-images',
|
||||
'memoir-manual',
|
||||
'minimal',
|
||||
'minimal-gvisor',
|
||||
]
|
||||
|
||||
// historyId is Mongo-or-numeric: a Mongo ObjectId (SaaS) or a small
|
||||
// Postgres-style integer id sent and kept as a string (Server Pro/CE). This
|
||||
// is exactly RequestParser's own HISTORY_ID_REGEX
|
||||
// (/^([0-9a-f]{24}|[1-9][0-9]{0,9})$/), which zz.projectHistoryId()
|
||||
// also models (shared with history-v1); web sends it as
|
||||
// options.historyId?.toString() either way, so it is never a JSON number.
|
||||
const historyIdSchema = zz.projectHistoryId()
|
||||
|
||||
const compileOptionsSchema = z.strictObject({
|
||||
metricsPath: z.string().optional(),
|
||||
metricsMethod: z.string().optional(),
|
||||
compiler: z.enum(VALID_COMPILERS).optional(),
|
||||
compileFromClsiCache: z.boolean().optional(),
|
||||
populateClsiCache: z.boolean().optional(),
|
||||
enablePdfCaching: z.boolean().optional(),
|
||||
pdfCachingMinChunkSize: z.number().optional(),
|
||||
enableCheckpoint: z.boolean().optional(),
|
||||
timeout: z.number().optional(),
|
||||
// Unlike compiler/buildId/editorId/historyId, imageName has no fixed
|
||||
// vocabulary: it is checked against settings.clsi.docker.allowedImages,
|
||||
// a deployment-specific allowlist (see CompileManager._isImageNameAllowed
|
||||
// for the equivalent check on the sync/wordcount routes below). Modeling
|
||||
// it as an enum would hard-code one deployment's image list into the
|
||||
// schema itself.
|
||||
imageName: z.string().optional(),
|
||||
draft: z.boolean().optional(),
|
||||
png2pdf: z.boolean().optional(),
|
||||
stopOnFirstError: z.boolean().optional(),
|
||||
check: z.string().optional(),
|
||||
// wire reality: web (ClsiManager.mjs) always sends an array of flag
|
||||
// strings; RequestParser's own check merely requires typeof === 'object'.
|
||||
flags: z.array(z.string()).optional(),
|
||||
compileGroup: zz.compileGroup().optional(),
|
||||
syncType: z.string().optional(),
|
||||
syncState: z.string().optional(),
|
||||
historyId: historyIdSchema.optional(),
|
||||
editorId: z.uuid().optional(),
|
||||
buildId: zz.buildId().optional(),
|
||||
clsiPerfVariant: z.enum(VALID_CLSI_PERF_VARIANTS).optional(),
|
||||
})
|
||||
|
||||
const resourceSchema = z.strictObject({
|
||||
path: zz.filepath(),
|
||||
// The only real caller, web's ClsiManager.mjs, sends `modified` as
|
||||
// `file.created?.getTime()` — an epoch-millisecond number, never an ISO
|
||||
// string.
|
||||
modified: z.number().optional(),
|
||||
url: z.url().optional(),
|
||||
fallbackURL: z.url().optional(),
|
||||
content: z.string().optional(),
|
||||
})
|
||||
|
||||
const compileRequestSchema = z.strictObject({
|
||||
options: compileOptionsSchema.optional(),
|
||||
resources: z.array(resourceSchema).optional(),
|
||||
rootResourcePath: zz.filepath().optional(),
|
||||
baseHistoryVersion: z.number().optional(),
|
||||
// Validated when loading them in editor-core (Snapshot.fromRaw /
|
||||
// Change.mustFromRaw); reuse its schemas rather than approximate the shape
|
||||
// locally. rawChangeOperations is RawOperation[][]: one operations array
|
||||
// per history change, combined with a synthetic timestamp by
|
||||
// HistoryResourceWriter.changesFromRawChangeOperations.
|
||||
rawSnapshot: editorCoreSchemas.rawSnapshot.optional(),
|
||||
rawChangeOperations: z
|
||||
.array(z.array(editorCoreSchemas.rawOperation))
|
||||
.optional(),
|
||||
globalBlobs: z.array(editorCoreSchemas.rawBlobHash).optional(),
|
||||
// v1 conversions/submissions; spliced into a filestore URL downstream, so
|
||||
// it is held to the same traversal rules as rootResourcePath.
|
||||
filestoreBlobPrefix: zz.filepath().optional(),
|
||||
})
|
||||
|
||||
export const compileRequestBodySchema = z.strictObject({
|
||||
compile: compileRequestSchema,
|
||||
})
|
||||
@@ -0,0 +1,11 @@
|
||||
# Demo document
|
||||
|
||||
This is a test markdown document for conversion testing.
|
||||
|
||||
## Section 1
|
||||
|
||||
Some content in section 1.
|
||||
|
||||
## Section 2
|
||||
|
||||
Some content in section 2.
|
||||
@@ -121,7 +121,7 @@ Hello world
|
||||
expect(
|
||||
Client.syncFromPdfWithImage(
|
||||
this.project_id,
|
||||
'main.tex',
|
||||
1,
|
||||
100,
|
||||
200,
|
||||
'something/evil:1337'
|
||||
|
||||
@@ -86,6 +86,59 @@ describe('Conversions', function () {
|
||||
})
|
||||
})
|
||||
|
||||
describe('markdown conversion', function () {
|
||||
before(async function () {
|
||||
await ClsiApp.ensureRunning()
|
||||
})
|
||||
|
||||
it('should convert file to LaTeX', async function () {
|
||||
const sourcePath = Path.join(
|
||||
import.meta.dirname,
|
||||
'../fixtures/conversion-source.md'
|
||||
)
|
||||
const outputStream = fs.createWriteStream(
|
||||
'/tmp/clsi_acceptance_tests_' + crypto.randomUUID() + '.zip'
|
||||
)
|
||||
const { stream } = await Client.convertDocument(sourcePath, 'markdown')
|
||||
await pipeline(stream, outputStream)
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
yauzl.open(outputStream.path, { lazyEntries: true }, (err, zipfile) => {
|
||||
if (err) {
|
||||
return reject(err)
|
||||
}
|
||||
zipfile.on('error', reject)
|
||||
zipfile.on('end', resolve)
|
||||
zipfile.readEntry()
|
||||
zipfile.on('entry', entry => {
|
||||
if (entry.fileName === 'main.tex') {
|
||||
zipfile.openReadStream(entry, (err, readStream) => {
|
||||
if (err) {
|
||||
return reject(err)
|
||||
}
|
||||
let data = ''
|
||||
readStream.on('data', chunk => {
|
||||
data += chunk.toString()
|
||||
})
|
||||
readStream.on('end', () => {
|
||||
try {
|
||||
expect(data).to.include('\\begin{document}')
|
||||
expect(data).to.include('Demo document')
|
||||
zipfile.readEntry()
|
||||
} catch (err) {
|
||||
reject(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
} else {
|
||||
reject(new Error('Unexpected file in zip: ' + entry.fileName))
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('project-to-document conversion (responseFormat=json)', function () {
|
||||
before(async function () {
|
||||
await ClsiApp.ensureRunning()
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
import Path from 'node:path'
|
||||
import fs from 'node:fs'
|
||||
import Client from './helpers/Client.js'
|
||||
import ClsiApp from './helpers/ClsiApp.js'
|
||||
import { expect } from 'chai'
|
||||
import {
|
||||
fetchNothing,
|
||||
fetchString,
|
||||
fetchJson,
|
||||
fetchStream,
|
||||
} from '@overleaf/fetch-utils'
|
||||
import Settings from '@overleaf/settings'
|
||||
import FormData from 'form-data'
|
||||
import { buffer } from 'node:stream/consumers'
|
||||
import { expectValidationError } from '@overleaf/validation-tools/testUtils.js'
|
||||
|
||||
const host = Settings.apis.clsi.url
|
||||
|
||||
// A syntactically valid Mongo ObjectId used for the per-user route tests
|
||||
// below (CompileManager only uses it to namespace the compile/output
|
||||
// directories, so any well-formed id works).
|
||||
const userId = '0123456789abcdef01234567'
|
||||
|
||||
function simpleProjectRequest() {
|
||||
return {
|
||||
resources: [
|
||||
{
|
||||
path: 'main.tex',
|
||||
content: `\
|
||||
\\documentclass{article}
|
||||
\\begin{document}
|
||||
Hello world
|
||||
\\end{document}\
|
||||
`,
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
describe('Misc and per-user routes', function () {
|
||||
before(async function () {
|
||||
await ClsiApp.ensureRunning()
|
||||
})
|
||||
|
||||
describe('status endpoint', function () {
|
||||
it('GET /project/:project_id/status should say OK', async function () {
|
||||
const projectId = Client.randomId()
|
||||
const body = await fetchString(`${host}/project/${projectId}/status`)
|
||||
expect(body).to.equal('OK')
|
||||
})
|
||||
|
||||
it('POST /project/:project_id/status should say OK', async function () {
|
||||
const projectId = Client.randomId()
|
||||
const body = await fetchString(`${host}/project/${projectId}/status`, {
|
||||
method: 'POST',
|
||||
})
|
||||
expect(body).to.equal('OK')
|
||||
})
|
||||
})
|
||||
|
||||
describe('per-user compile routes', function () {
|
||||
let projectId, body
|
||||
|
||||
before(async function () {
|
||||
projectId = Client.randomId()
|
||||
body = await fetchJson(
|
||||
`${host}/project/${projectId}/user/${userId}/compile`,
|
||||
{
|
||||
method: 'POST',
|
||||
json: { compile: simpleProjectRequest() },
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('should compile successfully', function () {
|
||||
expect(body.compile.status).to.equal('success')
|
||||
})
|
||||
|
||||
it('should include the per-user build URL in the output files', function () {
|
||||
const pdf = body.compile.outputFiles.find(f => f.type === 'pdf')
|
||||
expect(pdf.url).to.include(`/project/${projectId}/user/${userId}/build/`)
|
||||
})
|
||||
|
||||
it('should download the output.zip archive via the per-user route', async function () {
|
||||
const buildId = body.compile.outputFiles[0].build
|
||||
const res = await fetchNothing(
|
||||
`${host}/project/${projectId}/user/${userId}/build/${buildId}/output/output.zip`
|
||||
)
|
||||
expect(res.status).to.equal(200)
|
||||
})
|
||||
|
||||
it('should return a wordcount via the per-user route', async function () {
|
||||
const wc = await fetchJson(
|
||||
`${host}/project/${projectId}/user/${userId}/wordcount?file=main.tex`
|
||||
)
|
||||
expect(wc.texcount).to.exist
|
||||
})
|
||||
|
||||
it('should sync from code via the per-user route', async function () {
|
||||
const positions = await fetchJson(
|
||||
`${host}/project/${projectId}/user/${userId}/sync/code?file=main.tex&line=3&column=5`
|
||||
)
|
||||
expect(positions.pdf).to.exist
|
||||
})
|
||||
|
||||
it('should stop the compile via the per-user route', async function () {
|
||||
const res = await fetchNothing(
|
||||
`${host}/project/${projectId}/user/${userId}/compile/stop`,
|
||||
{ method: 'POST' }
|
||||
)
|
||||
expect(res.status).to.equal(204)
|
||||
})
|
||||
|
||||
it('should clear the cache via the per-user route', async function () {
|
||||
const res = await fetchNothing(
|
||||
`${host}/project/${projectId}/user/${userId}`,
|
||||
{ method: 'DELETE' }
|
||||
)
|
||||
expect(res.status).to.equal(204)
|
||||
})
|
||||
})
|
||||
|
||||
describe('output.zip download (project-only route)', function () {
|
||||
it('should download the archive for a project-only compile', async function () {
|
||||
const projectId = Client.randomId()
|
||||
const compileBody = await Client.compile(
|
||||
projectId,
|
||||
simpleProjectRequest()
|
||||
)
|
||||
const buildId = compileBody.compile.outputFiles[0].build
|
||||
const res = await fetchNothing(
|
||||
`${host}/project/${projectId}/build/${buildId}/output/output.zip`
|
||||
)
|
||||
expect(res.status).to.equal(200)
|
||||
})
|
||||
})
|
||||
|
||||
describe('legacy /convert/docx-to-latex route', function () {
|
||||
it('should convert a docx file without an explicit type query param', async function () {
|
||||
const sourcePath = Path.join(
|
||||
import.meta.dirname,
|
||||
'../fixtures/conversion-source.docx'
|
||||
)
|
||||
const formData = new FormData()
|
||||
formData.append('qqfile', fs.createReadStream(sourcePath))
|
||||
const stream = await fetchStream(`${host}/convert/docx-to-latex`, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
})
|
||||
const body = await buffer(stream)
|
||||
expect(body.length).to.be.greaterThan(0)
|
||||
// .zip archive — verify the PK\x03\x04 magic bytes
|
||||
expect(body[0]).to.equal(0x50)
|
||||
expect(body[1]).to.equal(0x4b)
|
||||
})
|
||||
})
|
||||
|
||||
describe('negative validation', function () {
|
||||
it('should reject a path-traversal rootResourcePath with 400', async function () {
|
||||
const projectId = Client.randomId()
|
||||
let error
|
||||
try {
|
||||
await Client.compile(projectId, {
|
||||
...simpleProjectRequest(),
|
||||
rootResourcePath: '../evil.tex',
|
||||
})
|
||||
expect.fail('should have thrown')
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
expectValidationError(error, 400, 'rootResourcePath')
|
||||
})
|
||||
|
||||
it('should reject a malformed user_id with 404', async function () {
|
||||
const projectId = Client.randomId()
|
||||
let error
|
||||
try {
|
||||
await fetchJson(
|
||||
`${host}/project/${projectId}/user/not-a-valid-user-id/wordcount?file=main.tex`
|
||||
)
|
||||
expect.fail('should have thrown')
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
expectValidationError(error, 404, 'user_id')
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -80,4 +80,19 @@ describe('pdf-to-jpeg conversion', function () {
|
||||
expect(response.status).to.equal(400)
|
||||
})
|
||||
})
|
||||
|
||||
describe('with a compileBackendClass query param', function () {
|
||||
it('returns 200', async function () {
|
||||
// v1's CLSI::PdfToJpeg always appends this (Rails.application.config
|
||||
// .conversion_compile_backend_class); clsi-lb already used it to
|
||||
// route to the right backend pool, so this instance's handler
|
||||
// doesn't need to read it again.
|
||||
const response = await Client.convertPdfToJpeg(
|
||||
FIXTURE_PDF,
|
||||
'thumbnail',
|
||||
'free'
|
||||
)
|
||||
expect(response.status).to.equal(200)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -18,7 +18,7 @@ Hello world
|
||||
request: {
|
||||
resources: [{ path: 'main.tex', content }],
|
||||
options: {
|
||||
compileGroup: 'simple-latex-file',
|
||||
compileGroup: 'alpha',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -27,7 +27,7 @@ Hello world
|
||||
request: {
|
||||
resources: [{ path: 'main.tex', content }],
|
||||
options: {
|
||||
compileGroup: 'simple-latex-file',
|
||||
compileGroup: 'alpha',
|
||||
imageName: 'quay.io/sharelatex/texlive-full:2017.1',
|
||||
},
|
||||
},
|
||||
@@ -42,7 +42,7 @@ Hello world
|
||||
enablePdfCaching: false,
|
||||
metricsPath: 'clsi-perf',
|
||||
metricsMethod: 'memoir-manual',
|
||||
compileGroup: 'clsi-perf', // only used by tests, not by the service
|
||||
compileGroup: 'priority',
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -5,6 +5,43 @@ import sinon from 'sinon'
|
||||
import ClsiApp from './helpers/ClsiApp.js'
|
||||
import { fetchString } from '@overleaf/fetch-utils'
|
||||
import Settings from '@overleaf/settings'
|
||||
import {
|
||||
handleValidationError,
|
||||
parseReq,
|
||||
z,
|
||||
zz,
|
||||
} from '@overleaf/validation-tools'
|
||||
|
||||
// Stands in for filestore's file-serving endpoints; this mock runs as its
|
||||
// own Express app in the same process as clsi's acceptance tests, so it's
|
||||
// subject to the same REQ_LOCKDOWN_MODE lockdown and needs its own schemas.
|
||||
const failParamsSchema = z.object({
|
||||
params: z.strictObject({
|
||||
times: z.string(),
|
||||
id: z.string(),
|
||||
}),
|
||||
})
|
||||
|
||||
const projectFileParamsSchema = z.object({
|
||||
params: z.strictObject({
|
||||
projectId: z.string(),
|
||||
fileId: z.string(),
|
||||
}),
|
||||
})
|
||||
|
||||
const bucketKeyParamsSchema = z.object({
|
||||
params: z.strictObject({
|
||||
bucket: z.string(),
|
||||
0: zz.filepath(),
|
||||
}),
|
||||
})
|
||||
|
||||
const staticFileParamsSchema = z.object({
|
||||
params: z.strictObject({
|
||||
random_id: z.string(),
|
||||
0: zz.filepath(),
|
||||
}),
|
||||
})
|
||||
|
||||
const Server = {
|
||||
run() {
|
||||
@@ -18,10 +55,13 @@ const Server = {
|
||||
app.get('/fail/:times/:id', (req, res) => {
|
||||
this.getFile(req.url)
|
||||
|
||||
const soFar = alreadyFailed.get(req.params.id) || 0
|
||||
const wanted = parseInt(req.params.times, 10)
|
||||
const {
|
||||
params: { times, id },
|
||||
} = parseReq(req, failParamsSchema)
|
||||
const soFar = alreadyFailed.get(id) || 0
|
||||
const wanted = parseInt(times, 10)
|
||||
if (soFar < wanted) {
|
||||
alreadyFailed.set(req.params.id, soFar + 1)
|
||||
alreadyFailed.set(id, soFar + 1)
|
||||
res.status(503).end()
|
||||
} else {
|
||||
res.send('THE CONTENT')
|
||||
@@ -35,20 +75,31 @@ const Server = {
|
||||
|
||||
app.get('/project/:projectId/file/:fileId', (req, res, next) => {
|
||||
this.getFile(req.url)
|
||||
res.send(`${req.params.projectId}:${req.params.fileId}`)
|
||||
const {
|
||||
params: { projectId, fileId },
|
||||
} = parseReq(req, projectFileParamsSchema)
|
||||
res.send(`${projectId}:${fileId}`)
|
||||
})
|
||||
|
||||
app.get('/bucket/:bucket/key/*', (req, res, next) => {
|
||||
this.getFile(req.url)
|
||||
res.send(`${req.params.bucket}:${req.params[0]}`)
|
||||
const {
|
||||
params: { bucket, 0: key },
|
||||
} = parseReq(req, bucketKeyParamsSchema)
|
||||
res.send(`${bucket}:${key}`)
|
||||
})
|
||||
|
||||
app.get('/:random_id/*', (req, res, next) => {
|
||||
this.getFile(req.url)
|
||||
req.url = `/${req.params[0]}`
|
||||
const {
|
||||
params: { 0: path },
|
||||
} = parseReq(req, staticFileParamsSchema)
|
||||
req.url = `/${path}`
|
||||
staticServer(req, res, next)
|
||||
})
|
||||
|
||||
app.use(handleValidationError)
|
||||
|
||||
Client.startFakeFilestoreApp(app)
|
||||
},
|
||||
|
||||
@@ -237,7 +288,7 @@ describe('Url Caching', function () {
|
||||
await Client.compile(this.project_id, this.request)
|
||||
|
||||
sinon.spy(Server, 'getFile')
|
||||
this.image_resource.modified = new Date(this.last_modified + 3000)
|
||||
this.image_resource.modified = this.last_modified + 3000
|
||||
|
||||
await Client.compile(this.project_id, this.request)
|
||||
})
|
||||
@@ -278,7 +329,7 @@ describe('Url Caching', function () {
|
||||
await Client.compile(this.project_id, this.request)
|
||||
|
||||
sinon.spy(Server, 'getFile')
|
||||
this.image_resource.modified = new Date(this.last_modified - 3000)
|
||||
this.image_resource.modified = this.last_modified - 3000
|
||||
|
||||
await Client.compile(this.project_id, this.request)
|
||||
})
|
||||
|
||||
@@ -33,14 +33,18 @@ function compile(projectId, data) {
|
||||
async function convertDocument(path, type) {
|
||||
const formData = new FormData()
|
||||
formData.append('qqfile', fs.createReadStream(path))
|
||||
// web's DocumentConversionManager.mjs always appends these two alongside
|
||||
// type, for clsi-lb backend routing -- sent here too so acceptance tests
|
||||
// exercise the same query shape real traffic uses.
|
||||
const url = new URL(`${host}/convert/document-to-latex`)
|
||||
url.searchParams.set('type', type)
|
||||
url.searchParams.set('compileBackendClass', 'e2e-tests')
|
||||
url.searchParams.set('compileGroup', 'standard')
|
||||
try {
|
||||
const stream = await fetchStream(
|
||||
`${host}/convert/document-to-latex?type=${type}`,
|
||||
{
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
}
|
||||
)
|
||||
const stream = await fetchStream(url.href, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
})
|
||||
return { status: 200, stream, body: null }
|
||||
} catch (err) {
|
||||
if (!err.response) throw err
|
||||
@@ -53,10 +57,15 @@ async function convertDocument(path, type) {
|
||||
}
|
||||
}
|
||||
|
||||
async function convertPdfToJpeg(path, mode) {
|
||||
async function convertPdfToJpeg(path, mode, compileBackendClass) {
|
||||
const formData = new FormData()
|
||||
formData.append('qqfile', await fsPromises.readFile(path), 'input.pdf')
|
||||
return await fetch(`${host}/convert/pdf-to-jpeg?mode=${mode}`, {
|
||||
const url = new URL(`${host}/convert/pdf-to-jpeg`)
|
||||
url.searchParams.set('mode', mode)
|
||||
if (compileBackendClass) {
|
||||
url.searchParams.set('compileBackendClass', compileBackendClass)
|
||||
}
|
||||
return await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: formData.getHeaders(),
|
||||
body: formData.getBuffer(),
|
||||
@@ -77,6 +86,11 @@ async function convertProjectToDocument(
|
||||
if (responseFormat) {
|
||||
url.searchParams.set('responseFormat', responseFormat)
|
||||
}
|
||||
// web's DocumentConversionManager.mjs always appends these two alongside
|
||||
// type/responseFormat, for clsi-lb backend routing -- sent here too so
|
||||
// acceptance tests exercise the same query shape real traffic uses.
|
||||
url.searchParams.set('compileBackendClass', 'e2e-tests')
|
||||
url.searchParams.set('compileGroup', 'standard')
|
||||
const opts = { method: 'POST', json: { compile: request } }
|
||||
if (responseFormat === 'json') {
|
||||
return await fetchJson(url.href, opts)
|
||||
@@ -215,7 +229,7 @@ async function compileDirectory(projectId, baseDirectory, directory) {
|
||||
resources.push({
|
||||
path: entity,
|
||||
url: `http://filestore/${directory}/${entity}`,
|
||||
modified: stat.mtime,
|
||||
modified: stat.mtime.getTime(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ describe('CompileController', () => {
|
||||
describe('compile', () => {
|
||||
beforeEach(ctx => {
|
||||
ctx.req.body = {
|
||||
compile: 'mock-body',
|
||||
compile: {},
|
||||
}
|
||||
ctx.req.params = { project_id: (ctx.project_id = 'project-id-123') }
|
||||
ctx.request = {
|
||||
|
||||
@@ -9,6 +9,21 @@ const MODULE_PATH = Path.join(
|
||||
'../../../app/js/ConversionController'
|
||||
)
|
||||
|
||||
// A minimal but schema-valid multer file object (zz.uploadedFile()), keyed
|
||||
// off the temp path used throughout these tests.
|
||||
function uploadedFile(path) {
|
||||
return {
|
||||
fieldname: 'qqfile',
|
||||
originalname: Path.basename(path),
|
||||
encoding: '7bit',
|
||||
mimetype: 'application/octet-stream',
|
||||
size: 1234,
|
||||
destination: Path.dirname(path),
|
||||
filename: Path.basename(path),
|
||||
path,
|
||||
}
|
||||
}
|
||||
|
||||
describe('ConversionController', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.conversionDir = '/path/to/conversion/result'
|
||||
@@ -131,7 +146,7 @@ describe('ConversionController', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.Settings.enablePandocConversions = false
|
||||
ctx.req = {
|
||||
file: { path: '/path/to/uploaded/file.docx' },
|
||||
file: uploadedFile('/path/to/uploaded/file.docx'),
|
||||
query: { type: 'docx' },
|
||||
}
|
||||
ctx.res.sendStatus = sinon.stub()
|
||||
@@ -157,20 +172,26 @@ describe('ConversionController', function () {
|
||||
describe('when conversionType is missing', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req = {
|
||||
file: { path: '/path/to/uploaded/file.docx' },
|
||||
file: uploadedFile('/path/to/uploaded/file.docx'),
|
||||
query: {},
|
||||
}
|
||||
ctx.res.sendStatus = sinon.stub()
|
||||
|
||||
await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
|
||||
try {
|
||||
await ctx.ConversionController.convertDocumentToLaTeX(
|
||||
ctx.req,
|
||||
ctx.res
|
||||
)
|
||||
} catch (err) {
|
||||
ctx.error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should remove the uploaded file', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.fs.unlink, ctx.req.file.path)
|
||||
})
|
||||
|
||||
it('should return 400', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.res.sendStatus, 400)
|
||||
it('should reject with a validation error', function (ctx) {
|
||||
expect(ctx.error).to.be.instanceOf(Error)
|
||||
})
|
||||
|
||||
it('should not call the conversion manager', function (ctx) {
|
||||
@@ -183,20 +204,26 @@ describe('ConversionController', function () {
|
||||
describe('when conversionType is unsupported', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req = {
|
||||
file: { path: '/path/to/uploaded/file.docx' },
|
||||
file: uploadedFile('/path/to/uploaded/file.docx'),
|
||||
query: { type: 'invalid' },
|
||||
}
|
||||
ctx.res.sendStatus = sinon.stub()
|
||||
|
||||
await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
|
||||
try {
|
||||
await ctx.ConversionController.convertDocumentToLaTeX(
|
||||
ctx.req,
|
||||
ctx.res
|
||||
)
|
||||
} catch (err) {
|
||||
ctx.error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should remove the uploaded file', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.fs.unlink, ctx.req.file.path)
|
||||
})
|
||||
|
||||
it('should return 400', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.res.sendStatus, 400)
|
||||
it('should reject with a validation error', function (ctx) {
|
||||
expect(ctx.error).to.be.instanceOf(Error)
|
||||
})
|
||||
|
||||
it('should not call the conversion manager', function (ctx) {
|
||||
@@ -209,7 +236,7 @@ describe('ConversionController', function () {
|
||||
describe('successfully', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req = {
|
||||
file: { path: '/path/to/uploaded/file.docx' },
|
||||
file: uploadedFile('/path/to/uploaded/file.docx'),
|
||||
query: { type: 'docx' },
|
||||
}
|
||||
|
||||
@@ -258,7 +285,7 @@ describe('ConversionController', function () {
|
||||
describe('with conversionType=markdown', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req = {
|
||||
file: { path: '/path/to/uploaded/file.md' },
|
||||
file: uploadedFile('/path/to/uploaded/file.md'),
|
||||
query: { type: 'markdown' },
|
||||
}
|
||||
|
||||
@@ -277,6 +304,30 @@ describe('ConversionController', function () {
|
||||
})
|
||||
})
|
||||
|
||||
describe('with web-supplied compileBackendClass/compileGroup query params', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req = {
|
||||
file: uploadedFile('/path/to/uploaded/file.md'),
|
||||
query: {
|
||||
type: 'markdown',
|
||||
compileBackendClass: 'premium',
|
||||
compileGroup: 'priority',
|
||||
},
|
||||
}
|
||||
|
||||
await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
|
||||
})
|
||||
|
||||
it('should not reject the request', function (ctx) {
|
||||
sinon.assert.calledWith(
|
||||
ctx.ConversionManager.promises.convertToLaTeXWithLock,
|
||||
sinon.match.string,
|
||||
ctx.req.file.path,
|
||||
'markdown'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('unsuccessfully', function () {
|
||||
describe('on streaming error', function () {
|
||||
it('should propagate the error and still clean up', async function (ctx) {
|
||||
@@ -287,7 +338,7 @@ describe('ConversionController', function () {
|
||||
res.setHeader = sinon.stub()
|
||||
|
||||
const req = {
|
||||
file: { path: '/path/to/uploaded/file.docx' },
|
||||
file: uploadedFile('/path/to/uploaded/file.docx'),
|
||||
query: { type: 'docx' },
|
||||
}
|
||||
|
||||
@@ -304,7 +355,7 @@ describe('ConversionController', function () {
|
||||
ctx.jsonStub = sinon.stub()
|
||||
ctx.res.status = sinon.stub().returns({ json: ctx.jsonStub })
|
||||
ctx.req = {
|
||||
file: { path: '/path/to/uploaded/file.docx' },
|
||||
file: uploadedFile('/path/to/uploaded/file.docx'),
|
||||
query: { type: 'docx' },
|
||||
}
|
||||
ctx.ConversionManager.promises.convertToLaTeXWithLock.rejects(
|
||||
@@ -339,7 +390,7 @@ describe('ConversionController', function () {
|
||||
ctx.jsonStub = sinon.stub()
|
||||
ctx.res.status = sinon.stub().returns({ json: ctx.jsonStub })
|
||||
ctx.req = {
|
||||
file: { path: '/path/to/uploaded/file.docx' },
|
||||
file: uploadedFile('/path/to/uploaded/file.docx'),
|
||||
query: { type: 'docx' },
|
||||
}
|
||||
ctx.ConversionManager.promises.convertToLaTeXWithLock.rejects(
|
||||
@@ -371,8 +422,11 @@ describe('ConversionController', function () {
|
||||
describe('convertProjectToDocument', function () {
|
||||
beforeEach(function (ctx) {
|
||||
ctx.req = {
|
||||
body: {},
|
||||
params: { project_id: 'test-project-id', user_id: 'test-user-id' },
|
||||
body: { compile: {} },
|
||||
params: {
|
||||
project_id: 'test-project-id',
|
||||
user_id: '0123456789abcdef01234567',
|
||||
},
|
||||
query: { type: 'docx' },
|
||||
}
|
||||
ctx.fs.stat.resolves(ctx.documentStat)
|
||||
@@ -405,17 +459,18 @@ describe('ConversionController', function () {
|
||||
describe('when an unsupported type is requested', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req.query = { type: 'unsupported' }
|
||||
ctx.res.sendStatus = sinon.stub()
|
||||
ctx.next = sinon.stub()
|
||||
|
||||
await ctx.ConversionController.convertProjectToDocument(
|
||||
ctx.req,
|
||||
ctx.res,
|
||||
sinon.stub()
|
||||
ctx.next
|
||||
)
|
||||
})
|
||||
|
||||
it('should return 400', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.res.sendStatus, 400)
|
||||
it('should call next with a validation error', function (ctx) {
|
||||
sinon.assert.calledOnce(ctx.next)
|
||||
expect(ctx.next.firstCall.args[0]).to.be.instanceOf(Error)
|
||||
})
|
||||
|
||||
it('should not sync resources or call the conversion manager', function (ctx) {
|
||||
@@ -446,7 +501,7 @@ describe('ConversionController', function () {
|
||||
sinon.assert.calledWith(
|
||||
ctx.HistoryResourceWriter.syncResourcesToDisk,
|
||||
'test-project-id',
|
||||
'test-user-id',
|
||||
'0123456789abcdef01234567',
|
||||
sinon.match({ png2pdf: false }),
|
||||
sinon.match(uuidDirPattern),
|
||||
sinon.match.object, // timings
|
||||
@@ -590,7 +645,7 @@ describe('ConversionController', function () {
|
||||
|
||||
describe('with conversionType=markdown', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req.query = { type: 'markdown', projectName: 'My_Project' }
|
||||
ctx.req.query = { type: 'markdown' }
|
||||
ctx.fs.stat.resolves(ctx.documentStat)
|
||||
|
||||
await ctx.ConversionController.convertProjectToDocument(
|
||||
@@ -619,6 +674,32 @@ describe('ConversionController', function () {
|
||||
})
|
||||
})
|
||||
|
||||
describe('with web-supplied compileBackendClass/compileGroup query params', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req.query = {
|
||||
type: 'docx',
|
||||
compileBackendClass: 'premium',
|
||||
compileGroup: 'priority',
|
||||
}
|
||||
|
||||
await ctx.ConversionController.convertProjectToDocument(
|
||||
ctx.req,
|
||||
ctx.res,
|
||||
sinon.stub()
|
||||
)
|
||||
})
|
||||
|
||||
it('should not reject the request', function (ctx) {
|
||||
sinon.assert.calledWith(
|
||||
ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock,
|
||||
sinon.match.string,
|
||||
sinon.match.string,
|
||||
'main.tex',
|
||||
'docx'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('with conversionType=html', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req.query = { type: 'html' }
|
||||
|
||||
@@ -38,8 +38,8 @@ describe('OutputController', () => {
|
||||
ctx.req = {
|
||||
params: {
|
||||
project_id: 'project-id-123',
|
||||
user_id: 'user-id-123',
|
||||
build_id: 'build-id-123',
|
||||
user_id: '012345678901234567890123',
|
||||
build_id: '1234-5678',
|
||||
},
|
||||
query: {
|
||||
files: ['output.tex'],
|
||||
@@ -86,8 +86,8 @@ describe('OutputController', () => {
|
||||
ctx.req = {
|
||||
params: {
|
||||
project_id: 'project-id-123',
|
||||
user_id: 'user-id-123',
|
||||
build_id: 'build-id-123',
|
||||
user_id: '012345678901234567890123',
|
||||
build_id: '1234-5678',
|
||||
},
|
||||
query: {
|
||||
files: ['output.tex'],
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { compileRequestBodySchema } from '../../../app/js/schemas.js'
|
||||
|
||||
describe('schemas', function () {
|
||||
describe('compileRequestBodySchema', function () {
|
||||
it('accepts a compileGroup from the enum', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: { options: { compileGroup: 'priority' } },
|
||||
})
|
||||
expect(result.success).to.equal(true)
|
||||
})
|
||||
|
||||
it('accepts a missing compileGroup', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: { options: {} },
|
||||
})
|
||||
expect(result.success).to.equal(true)
|
||||
})
|
||||
|
||||
it('rejects a compileGroup outside the enum', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: { options: { compileGroup: 'clsi-perf' } },
|
||||
})
|
||||
expect(result.success).to.equal(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('compileRequestBodySchema globalBlobs', function () {
|
||||
it('accepts an array of valid blob hashes', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: { globalBlobs: ['a'.repeat(40), 'b'.repeat(40)] },
|
||||
})
|
||||
expect(result.success).to.equal(true)
|
||||
})
|
||||
|
||||
it('accepts a missing globalBlobs', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: {},
|
||||
})
|
||||
expect(result.success).to.equal(true)
|
||||
})
|
||||
|
||||
it('rejects a globalBlobs entry that is not a valid blob hash', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: { globalBlobs: ['not-a-valid-hash'] },
|
||||
})
|
||||
expect(result.success).to.equal(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('compileRequestBodySchema resources', function () {
|
||||
it('accepts a resource with a valid url and fallbackURL', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: {
|
||||
resources: [
|
||||
{
|
||||
path: 'main.tex',
|
||||
url: 'http://example.com/main.tex',
|
||||
fallbackURL: 'http://fallback.example.com/main.tex',
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
expect(result.success).to.equal(true)
|
||||
})
|
||||
|
||||
it('rejects a resource with a non-URL url', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: {
|
||||
resources: [{ path: 'main.tex', url: 'not-a-url' }],
|
||||
},
|
||||
})
|
||||
expect(result.success).to.equal(false)
|
||||
})
|
||||
|
||||
it('rejects a resource with a non-URL fallbackURL', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: {
|
||||
resources: [
|
||||
{
|
||||
path: 'main.tex',
|
||||
url: 'http://example.com/main.tex',
|
||||
fallbackURL: 'not-a-url',
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
expect(result.success).to.equal(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('compileRequestBodySchema clsiPerfVariant', function () {
|
||||
it('accepts a clsiPerfVariant from the enum', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: { options: { clsiPerfVariant: 'minimal-gvisor' } },
|
||||
})
|
||||
expect(result.success).to.equal(true)
|
||||
})
|
||||
|
||||
it('accepts a missing clsiPerfVariant', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: { options: {} },
|
||||
})
|
||||
expect(result.success).to.equal(true)
|
||||
})
|
||||
|
||||
it('rejects a clsiPerfVariant outside the enum', function () {
|
||||
const result = compileRequestBodySchema.safeParse({
|
||||
compile: { options: { clsiPerfVariant: '../../etc' } },
|
||||
})
|
||||
expect(result.success).to.equal(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user