diff --git a/libraries/validation-tools/handleValidationError.js b/libraries/validation-tools/handleValidationError.js index 8f789a83c7..a5038448eb 100644 --- a/libraries/validation-tools/handleValidationError.js +++ b/libraries/validation-tools/handleValidationError.js @@ -1,17 +1,17 @@ const { isZodErrorLike, fromError } = require('zod-validation-error') -/** - * @typedef {import('express').ErrorRequestHandler} ErrorRequestHandler - */ -const handleValidationError = [ - /** @type {ErrorRequestHandler} */ - (err, req, res, next) => { - if (!isZodErrorLike(err)) { - return next(err) - } +function createHandleValidationError(statusCode = 400) { + return [ + (err, req, res, next) => { + if (!isZodErrorLike(err)) { + return next(err) + } - res.status(400).json({ ...fromError(err), statusCode: 400 }) - }, -] + res.status(statusCode).json({ ...fromError(err), statusCode }) + }, + ] +} -module.exports = { handleValidationError } +const handleValidationError = createHandleValidationError(400) + +module.exports = { handleValidationError, createHandleValidationError } diff --git a/libraries/validation-tools/index.js b/libraries/validation-tools/index.js index b996e1905c..c41b37d0b5 100644 --- a/libraries/validation-tools/index.js +++ b/libraries/validation-tools/index.js @@ -3,7 +3,10 @@ const { z } = require('zod') const { zz } = require('./zodHelpers') const { validateReq } = require('./validateReq') const { validateSchema } = require('./validateSchema') -const { handleValidationError } = require('./handleValidationError') +const { + handleValidationError, + createHandleValidationError, +} = require('./handleValidationError') module.exports = { z, @@ -11,5 +14,6 @@ module.exports = { validateSchema, validateReq, handleValidationError, + createHandleValidationError, ParamsError, } diff --git a/services/history-v1/api/controllers/expressify.js b/services/history-v1/api/controllers/expressify.js deleted file mode 100644 index 5eee15fbe6..0000000000 --- a/services/history-v1/api/controllers/expressify.js +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Turn an async function into an Express middleware - */ -function expressify(fn) { - return (req, res, next) => { - fn(req, res, next).catch(next) - } -} - -module.exports = expressify diff --git a/services/history-v1/api/controllers/health_checks.js b/services/history-v1/api/controllers/health_checks.js index e9f7176b9f..5830c7c790 100644 --- a/services/history-v1/api/controllers/health_checks.js +++ b/services/history-v1/api/controllers/health_checks.js @@ -1,5 +1,5 @@ const logger = require('@overleaf/logger') -const expressify = require('./expressify') +const { expressify } = require('@overleaf/promise-utils') const { mongodb } = require('../../storage') async function status(req, res) { diff --git a/services/history-v1/api/controllers/project_import.js b/services/history-v1/api/controllers/project_import.js index 02fb793c87..d4cb94886b 100644 --- a/services/history-v1/api/controllers/project_import.js +++ b/services/history-v1/api/controllers/project_import.js @@ -27,16 +27,36 @@ const persistBuffer = storage.persistBuffer const InvalidChangeError = storage.InvalidChangeError const render = require('./render') +const { validateReq } = require('@overleaf/validation-tools') +const schemas = require('../schema') const Rollout = require('../app/rollout') const redisBackend = require('../../storage/lib/chunk_store/redis') const rollout = new Rollout(config) rollout.report(logger) // display the rollout configuration in the logs +function getParam(req, name, location = 'path') { + switch (location) { + case 'path': + return req.params?.[name] + case 'query': + return req.query?.[name] + case 'body': + if (name === 'body') { + return req.body + } + if (req.body?.[name] !== undefined) { + return req.body[name] + } + return undefined + default: + return undefined + } +} async function importSnapshot(req, res) { - const projectId = req.swagger.params.project_id.value - const rawSnapshot = req.swagger.params.snapshot.value - + const { params, body } = validateReq(req, schemas.importSnapshot) + const projectId = params.project_id + const rawSnapshot = getParam({ body }, 'snapshot', 'body') ?? body let snapshot try { @@ -62,10 +82,11 @@ async function importSnapshot(req, res) { } async function importChanges(req, res, next) { - const projectId = req.swagger.params.project_id.value - const rawChanges = req.swagger.params.changes.value - const endVersion = req.swagger.params.end_version.value - const returnSnapshot = req.swagger.params.return_snapshot.value || 'none' + const { params, query, body } = validateReq(req, schemas.importChanges) + const projectId = params.project_id + const rawChanges = getParam({ body }, 'changes', 'body') ?? body + const endVersion = query.end_version + const returnSnapshot = query.return_snapshot ?? 'none' let changes @@ -156,7 +177,8 @@ async function importChanges(req, res, next) { } async function flushChanges(req, res, next) { - const projectId = req.swagger.params.project_id.value + const { params } = validateReq(req, schemas.flushChanges) + const projectId = params.project_id // Use the same limits importChanges, since these are passed to persistChanges const farFuture = new Date() farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000) @@ -179,7 +201,8 @@ async function flushChanges(req, res, next) { } async function expireProject(req, res, next) { - const projectId = req.swagger.params.project_id.value + const { params } = validateReq(req, schemas.expireProject) + const projectId = params.project_id await redisBackend.expireProject(projectId) res.status(HTTPStatus.OK).end() } diff --git a/services/history-v1/api/controllers/projects.js b/services/history-v1/api/controllers/projects.js index 455cfeafb5..b8d567c6af 100644 --- a/services/history-v1/api/controllers/projects.js +++ b/services/history-v1/api/controllers/projects.js @@ -8,6 +8,8 @@ const fs = require('node:fs') const { promisify } = require('node:util') const config = require('config') const OError = require('@overleaf/o-error') +const { expressify } = require('@overleaf/promise-utils') +const { validateReq } = require('@overleaf/validation-tools') const logger = require('@overleaf/logger') const { Chunk, ChunkResponse, Blob } = require('overleaf-editor-core') @@ -23,7 +25,7 @@ const { } = require('../../storage') const render = require('./render') -const expressify = require('./expressify') +const schemas = require('../schema') const withTmpDir = require('./with_tmp_dir') const StreamSizeLimit = require('./stream_size_limit') const { getProjectBlobsBatch } = require('../../storage/lib/blob_store') @@ -33,7 +35,8 @@ const { getChunkMetadataForVersion } = require('../../storage/lib/chunk_store') const pipeline = promisify(Stream.pipeline) async function initializeProject(req, res, next) { - let projectId = req.swagger.params.body.value.projectId + const { body } = validateReq(req, schemas.initializeProject) + let projectId = body?.projectId try { projectId = await chunkStore.initializeProject(projectId) res.status(HTTPStatus.OK).json({ projectId }) @@ -48,7 +51,8 @@ async function initializeProject(req, res, next) { } async function getLatestContent(req, res, next) { - const projectId = req.swagger.params.project_id.value + const { params } = validateReq(req, schemas.getLatestContent) + const projectId = params.project_id const blobStore = new BlobStore(projectId) const chunk = await chunkStore.loadLatest(projectId) const snapshot = chunk.getSnapshot() @@ -58,8 +62,9 @@ async function getLatestContent(req, res, next) { } async function getContentAtVersion(req, res, next) { - const projectId = req.swagger.params.project_id.value - const version = req.swagger.params.version.value + const { params } = validateReq(req, schemas.getContentAtVersion) + const projectId = params.project_id + const version = params.version const blobStore = new BlobStore(projectId) const snapshot = await getSnapshotAtVersion(projectId, version) await snapshot.loadFiles('eager', blobStore) @@ -67,7 +72,8 @@ async function getContentAtVersion(req, res, next) { } async function getLatestHashedContent(req, res, next) { - const projectId = req.swagger.params.project_id.value + const { params } = validateReq(req, schemas.getLatestHashedContent) + const projectId = params.project_id const blobStore = new HashCheckBlobStore(new BlobStore(projectId)) const chunk = await chunkStore.loadLatest(projectId) const snapshot = chunk.getSnapshot() @@ -78,7 +84,8 @@ async function getLatestHashedContent(req, res, next) { } async function getLatestHistory(req, res, next) { - const projectId = req.swagger.params.project_id.value + const { params } = validateReq(req, schemas.getLatestHistory) + const projectId = params.project_id try { const chunk = await chunkStore.loadLatest(projectId) const chunkResponse = new ChunkResponse(chunk) @@ -93,8 +100,9 @@ async function getLatestHistory(req, res, next) { } async function getLatestHistoryRaw(req, res, next) { - const projectId = req.swagger.params.project_id.value - const readOnly = req.swagger.params.readOnly.value + const { params, query } = validateReq(req, schemas.getLatestHistoryRaw) + const projectId = params.project_id + const readOnly = query.readOnly try { const { startVersion, endVersion, endTimestamp } = await chunkStore.getLatestChunkMetadata(projectId, { readOnly }) @@ -113,8 +121,9 @@ async function getLatestHistoryRaw(req, res, next) { } async function getHistory(req, res, next) { - const projectId = req.swagger.params.project_id.value - const version = req.swagger.params.version.value + const { params } = validateReq(req, schemas.getHistory) + const projectId = params.project_id + const version = params.version try { const chunk = await chunkStore.loadAtVersion(projectId, version) const chunkResponse = new ChunkResponse(chunk) @@ -129,8 +138,9 @@ async function getHistory(req, res, next) { } async function getHistoryBefore(req, res, next) { - const projectId = req.swagger.params.project_id.value - const timestamp = req.swagger.params.timestamp.value + const { params } = validateReq(req, schemas.getHistoryBefore) + const projectId = params.project_id + const timestamp = params.timestamp try { const chunk = await chunkStore.loadAtTimestamp(projectId, timestamp) const chunkResponse = new ChunkResponse(chunk) @@ -148,8 +158,10 @@ async function getHistoryBefore(req, res, next) { * Get all changes since the beginning of history or since a given version */ async function getChanges(req, res, next) { - const projectId = req.swagger.params.project_id.value - const since = req.swagger.params.since.value ?? 0 + const { params, query } = validateReq(req, schemas.getChanges) + const projectId = params.project_id + const sinceParam = query.since + const since = sinceParam == null ? 0 : sinceParam if (since < 0) { // Negative values would cause an infinite loop @@ -175,8 +187,9 @@ async function getChanges(req, res, next) { } async function getZip(req, res, next) { - const projectId = req.swagger.params.project_id.value - const version = req.swagger.params.version.value + const { params } = validateReq(req, schemas.getZip) + const projectId = params.project_id + const version = params.version const blobStore = new BlobStore(projectId) let snapshot @@ -202,8 +215,9 @@ async function getZip(req, res, next) { } async function createZip(req, res, next) { - const projectId = req.swagger.params.project_id.value - const version = req.swagger.params.version.value + const { params } = validateReq(req, schemas.createZip) + const projectId = params.project_id + const version = params.version try { const snapshot = await getSnapshotAtVersion(projectId, version) const zipUrl = await zipStore.getSignedUrl(projectId, version) @@ -222,7 +236,8 @@ async function createZip(req, res, next) { } async function deleteProject(req, res, next) { - const projectId = req.swagger.params.project_id.value + const { params } = validateReq(req, schemas.deleteProject) + const projectId = params.project_id const blobStore = new BlobStore(projectId) await Promise.all([ @@ -234,8 +249,9 @@ async function deleteProject(req, res, next) { } async function createProjectBlob(req, res, next) { - const projectId = req.swagger.params.project_id.value - const expectedHash = req.swagger.params.hash.value + const { params } = validateReq(req, schemas.createProjectBlob) + const projectId = params.project_id + const expectedHash = params.hash const maxUploadSize = parseInt(config.get('maxFileUploadSize'), 10) await withTmpDir('blob-', async tmpDir => { @@ -271,8 +287,9 @@ async function createProjectBlob(req, res, next) { } async function headProjectBlob(req, res) { - const projectId = req.swagger.params.project_id.value - const hash = req.swagger.params.hash.value + const { params } = validateReq(req, schemas.headProjectBlob) + const projectId = params.project_id + const hash = params.hash const blobStore = new BlobStore(projectId) const blob = await blobStore.getBlob(hash) @@ -283,7 +300,6 @@ async function headProjectBlob(req, res) { res.status(404).end() } } - // Support simple, singular ranges starting from zero only, up-to 2MB = 2_000_000, 7 digits const RANGE_HEADER = /^bytes=(\d{1,7})-(\d{1,7})$/ @@ -304,13 +320,19 @@ function _getRangeOpts(header) { } async function getProjectBlob(req, res, next) { - const projectId = req.swagger.params.project_id.value - const hash = req.swagger.params.hash.value - const opts = _getRangeOpts(req.swagger.params.range.value || '') + const { params, headers } = validateReq(req, schemas.getProjectBlob) + const projectId = params.project_id + const hash = params.hash + const rangeHeader = headers.range || '' + const opts = _getRangeOpts(rangeHeader) const blobStore = new BlobStore(projectId) logger.debug({ projectId, hash }, 'getProjectBlob started') try { + if (req.method === 'HEAD') { + return await headProjectBlob(req, res) + } + let stream try { if (opts) { @@ -363,9 +385,10 @@ async function getProjectBlob(req, res, next) { } async function copyProjectBlob(req, res, next) { - const sourceProjectId = req.swagger.params.copyFrom.value - const targetProjectId = req.swagger.params.project_id.value - const blobHash = req.swagger.params.hash.value + const { params, query } = validateReq(req, schemas.copyProjectBlob) + const sourceProjectId = query.copyFrom + const targetProjectId = params.project_id + const blobHash = params.hash // Check that blob exists in source project const sourceBlobStore = new BlobStore(sourceProjectId) const targetBlobStore = new BlobStore(targetProjectId) @@ -427,8 +450,9 @@ function sumUpByteLength(blobs) { } async function getBlobStats(req, res) { - const projectId = req.swagger.params.project_id.value - const blobHashes = req.swagger.params.body.value.blobHashes || [] + const { params, body } = validateReq(req, schemas.getBlobStats) + const projectId = params.project_id + const blobHashes = body.blobHashes || [] for (const hash of blobHashes) { assert.blobHash(hash, 'bad hash') } @@ -451,7 +475,8 @@ async function getBlobStats(req, res) { } async function getProjectBlobsStats(req, res) { - const projectIds = req.swagger.params.body.value.projectIds + const { body } = validateReq(req, schemas.getProjectBlobsStats) + const projectIds = body.projectIds const { blobs } = await getProjectBlobsBatch( projectIds.map(id => { if (assert.POSTGRES_ID_REGEXP.test(id)) { diff --git a/services/history-v1/api/middleware/security.js b/services/history-v1/api/middleware/security.js index 582ffc62c4..0f9568c088 100644 --- a/services/history-v1/api/middleware/security.js +++ b/services/history-v1/api/middleware/security.js @@ -48,6 +48,19 @@ function setupSSL(app) { exports.setupSSL = setupSSL +function setupBasicHttpAuthForSwaggerDocs(app) { + app.use('/docs', function (req, res, next) { + if (hasValidBasicAuthCredentials(req)) { + return next() + } + + res.header('WWW-Authenticate', 'Basic realm="Application"') + res.status(HTTPStatus.UNAUTHORIZED).end() + }) +} + +exports.setupBasicHttpAuthForSwaggerDocs = setupBasicHttpAuthForSwaggerDocs + function configureJWTAuth(mode = 'jwt') { return function handleJWTAuth(req, res, next) { if (hasValidBasicAuthCredentials(req)) { diff --git a/services/history-v1/api/schema.js b/services/history-v1/api/schema.js index 993ea6a417..0effaeb4ad 100644 --- a/services/history-v1/api/schema.js +++ b/services/history-v1/api/schema.js @@ -1,6 +1,6 @@ 'use strict' -const { z } = require('@overleaf/validation-tools') +const { z, zz } = require('@overleaf/validation-tools') const Blob = require('overleaf-editor-core').Blob const hexHashPattern = new RegExp(Blob.HEX_HASH_RX_STRING) @@ -20,26 +20,38 @@ const v2DocVersionsSchema = z.object({ v: z.number().int().optional(), }) -const operationSchema = z.object({ - pathname: z.string().optional(), - newPathname: z.string().optional(), - blob: z - .object({ - hash: z.string(), - }) - .optional(), - textOperation: z.array(z.any()).optional(), - file: fileSchema.optional(), -}) +const operationSchema = z + .object({ + pathname: z.string().optional(), + newPathname: z.string().optional(), + blob: z + .object({ + hash: z.string(), + }) + .optional(), + textOperation: z.array(z.any()).optional(), + file: fileSchema.optional(), + contentHash: z.string().optional(), + }) + .passthrough() -const changeSchema = z.object({ - timestamp: z.string(), - operations: z.array(operationSchema), - authors: z.array(z.number().int().nullable()).optional(), - v2Authors: z.array(z.string().nullable()).optional(), - projectVersion: z.string().optional(), - v2DocVersions: z.record(v2DocVersionsSchema).optional(), -}) +const originSchema = z + .object({ + kind: z.string().optional(), + }) + .passthrough() + +const changeSchema = z + .object({ + timestamp: z.string(), + operations: z.array(operationSchema), + authors: z.array(z.number().int().nullable()).optional(), + v2Authors: z.array(z.string().nullable()).optional(), + origin: originSchema.optional(), + projectVersion: z.string().optional(), + v2DocVersions: z.record(z.string(), v2DocVersionsSchema).optional(), + }) + .passthrough() const schemas = { projectId: z.object({ @@ -138,7 +150,7 @@ const schemas = { project_id: z.string(), }), query: z.object({ - readOnly: z.boolean().optional(), + readOnly: z.coerce.boolean().optional(), }), }), @@ -165,7 +177,7 @@ const schemas = { getHistoryBefore: z.object({ params: z.object({ project_id: z.string(), - timestamp: z.iso.datetime(), + timestamp: zz.datetime(), }), }), diff --git a/services/history-v1/app.js b/services/history-v1/app.js index f5997604e7..a3bbb1b0ad 100644 --- a/services/history-v1/app.js +++ b/services/history-v1/app.js @@ -7,24 +7,27 @@ require('@overleaf/metrics/initialize') const config = require('config') const Events = require('node:events') -const BPromise = require('bluebird') const express = require('express') const helmet = require('helmet') const HTTPStatus = require('http-status') const logger = require('@overleaf/logger') const Metrics = require('@overleaf/metrics') const bodyParser = require('body-parser') -const swaggerTools = require('swagger-tools') -const swaggerDoc = require('./api/swagger') -const security = require('./api/app/security') +const security = require('./api/middleware/security') const healthChecks = require('./api/controllers/health_checks') const { mongodb, loadGlobalBlobs } = require('./storage') -const path = require('node:path') +const projectsRoutes = require('./api/routes/projects') +const projectImportRoutes = require('./api/routes/project_import') +const { createHandleValidationError } = require('@overleaf/validation-tools') Events.setMaxListeners(20) const app = express() module.exports = app +const handleValidationError = createHandleValidationError( + HTTPStatus.UNPROCESSABLE_ENTITY +) + logger.initialize('history-v1') Metrics.open_sockets.monitor() Metrics.injectMetricsRoute(app) @@ -57,23 +60,9 @@ app.get('/', function (req, res) { app.get('/status', healthChecks.status) app.get('/health_check', healthChecks.healthCheck) -function setupSwagger() { - return new BPromise(function (resolve) { - swaggerTools.initializeMiddleware(swaggerDoc, function (middleware) { - app.use(middleware.swaggerMetadata()) - app.use(middleware.swaggerSecurity(security.getSwaggerHandlers())) - app.use(middleware.swaggerValidator()) - app.use( - middleware.swaggerRouter({ - controllers: path.join(__dirname, 'api/controllers'), - useStubs: app.get('env') === 'development', - }) - ) - app.use(middleware.swaggerUi()) - resolve() - }) - }) -} +app.get('/docs', function (req, res) { + res.send('OK') +}) function setupErrorHandling() { app.use(function (req, res, next) { @@ -82,40 +71,10 @@ function setupErrorHandling() { return next(err) }) - // Handle Swagger errors. - app.use(function (err, req, res, next) { - const projectId = req.swagger?.params?.project_id?.value - if (res.headersSent) { - return next(err) - } - - if (err.code === 'SCHEMA_VALIDATION_FAILED') { - logger.error({ err, projectId }, err.message) - return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json(err.results) - } - if (err.code === 'INVALID_TYPE' || err.code === 'PATTERN') { - logger.error({ err, projectId }, err.message) - return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json({ - message: 'invalid type: ' + err.paramName, - }) - } - if (err.code === 'ENUM_MISMATCH') { - logger.warn({ err, projectId }, err.message) - return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json({ - message: 'invalid enum value: ' + err.paramName, - }) - } - if (err.code === 'REQUIRED') { - logger.warn({ err, projectId }, err.message) - return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json({ - message: err.message, - }) - } - next(err) - }) + app.use(handleValidationError) app.use(function (err, req, res, next) { - const projectId = req.swagger?.params?.project_id?.value + const projectId = req.params?.project_id || req.body?.projectId logger.error({ err, projectId }, err.message) if (res.headersSent) { @@ -127,6 +86,10 @@ function setupErrorHandling() { // 200, notably some InternalErrors and TimeoutErrors, so we have to guard // against that. We also check `status`, but `statusCode` is preferred. const statusCode = err.statusCode || err.status + if (err.headers) { + res.set(err.headers) + } + if (statusCode && statusCode >= 400 && statusCode < 600) { res.status(statusCode) } else { @@ -147,7 +110,8 @@ app.setup = async function appSetup() { await loadGlobalBlobs() logger.info('Global blobs loaded') app.use(helmet()) - await setupSwagger() + app.use('/api', projectsRoutes) + app.use('/api', projectImportRoutes) setupErrorHandling() } diff --git a/services/history-v1/backup-deletion-app.mjs b/services/history-v1/backup-deletion-app.mjs index 81b2b5b8b9..53aa298aee 100644 --- a/services/history-v1/backup-deletion-app.mjs +++ b/services/history-v1/backup-deletion-app.mjs @@ -7,7 +7,7 @@ import { promisify } from 'node:util' import express from 'express' import logger from '@overleaf/logger' import Metrics from '@overleaf/metrics' -import { hasValidBasicAuthCredentials } from './api/app/security.js' +import { hasValidBasicAuthCredentials } from './api/middleware/security.js' import { deleteProjectBackupCb, healthCheck, diff --git a/services/history-v1/test/acceptance/js/api/auth.test.js b/services/history-v1/test/acceptance/js/api/auth.test.js index 65c9219ae2..3b216cbcf0 100644 --- a/services/history-v1/test/acceptance/js/api/auth.test.js +++ b/services/history-v1/test/acceptance/js/api/auth.test.js @@ -1,12 +1,11 @@ const config = require('config') -const fetch = require('node-fetch') const sinon = require('sinon') const { expect } = require('chai') +const nodeFetch = require('node-fetch') const cleanup = require('../storage/support/cleanup') const expectResponse = require('./support/expect_response') const fixtures = require('../storage/support/fixtures') -const HTTPStatus = require('http-status') const testServer = require('./support/test_server') describe('auth', function () { @@ -18,47 +17,26 @@ describe('auth', function () { }) afterEach(sinon.restore) - it('renders 401 on swagger docs endpoint without auth', async function () { - const response = await fetch(testServer.url('/docs')) - expect(response.status).to.equal(HTTPStatus.UNAUTHORIZED) - expect(response.headers.get('www-authenticate')).to.match(/^Basic/) - }) + it('protects /docs with basic auth', async function () { + const url = testServer.url('/docs') - it('renders swagger docs endpoint with auth', async function () { - const response = await fetch(testServer.url('/docs'), { - headers: { - Authorization: testServer.basicAuthHeader, - }, + const unauthenticatedResponse = await nodeFetch(url) + expect(unauthenticatedResponse.status).to.equal(401) + expect(unauthenticatedResponse.headers.get('www-authenticate')).to.match( + /^Basic/ + ) + + const badHeader = + 'Basic ' + Buffer.from('staging:wrong-password').toString('base64') + const badPasswordResponse = await nodeFetch(url, { + headers: { Authorization: badHeader }, }) - expect(response.ok).to.be.true - }) + expect(badPasswordResponse.status).to.equal(401) - it('takes an old basic auth password during a password change', async function () { - setMockConfig('basicHttpAuth.oldPassword', 'foo') - - // Primary should still work. - const response1 = await fetch(testServer.url('/docs'), { - headers: { - Authorization: testServer.basicAuthHeader, - }, + const validResponse = await nodeFetch(url, { + headers: { Authorization: testServer.basicAuthHeader }, }) - expect(response1.ok).to.be.true - - // Old password should also work. - const response2 = await fetch(testServer.url('/docs'), { - headers: { - Authorization: 'Basic ' + Buffer.from('staging:foo').toString('base64'), - }, - }) - expect(response2.ok).to.be.true - - // Incorrect password should not work. - const response3 = await fetch(testServer.url('/docs'), { - header: { - Authorization: 'Basic ' + Buffer.from('staging:bar').toString('base64'), - }, - }) - expect(response3.status).to.equal(HTTPStatus.UNAUTHORIZED) + expect(validResponse.status).to.equal(200) }) it('renders 401 on ProjectImport endpoints', async function () { diff --git a/services/history-v1/test/acceptance/js/api/project_updates.test.js b/services/history-v1/test/acceptance/js/api/project_updates.test.js index f50f3677b5..2ec2e6cba7 100644 --- a/services/history-v1/test/acceptance/js/api/project_updates.test.js +++ b/services/history-v1/test/acceptance/js/api/project_updates.test.js @@ -449,6 +449,70 @@ describe('history import', function () { .catch(expectResponse.unprocessableEntity) }) + it('imports changes with git-bridge origin', function () { + const testProjectId = '1' + const testFilePathname = 'git.tex' + const testFile = File.fromHash(File.EMPTY_FILE_HASH) + const testGitOrigin = Origin.fromRaw({ + kind: 'git-bridge', + }) + + let testSnapshot + + return fetch( + testServer.url( + `/api/projects/${testProjectId}/blobs/${File.EMPTY_FILE_HASH}` + ), + { + method: 'PUT', + body: fs.createReadStream(testFiles.path('empty.tex')), + headers: { + Authorization: testServer.basicAuthHeader, + }, + } + ) + .then(response => { + expect(response.ok).to.be.true + }) + .then(() => { + testSnapshot = new Snapshot() + testSnapshot.addFile(testFilePathname, testFile) + return basicAuthClient.apis.ProjectImport.importSnapshot1({ + project_id: testProjectId, + snapshot: testSnapshot.toRaw(), + }) + }) + .then(response => { + expect(response.obj.projectId).to.equal(testProjectId) + }) + .then(() => { + const changes = [ + makeChange(Operation.addFile(testFilePathname, testFile)), + ] + changes[0].setOrigin(testGitOrigin) + return basicAuthClient.apis.ProjectImport.importChanges1({ + project_id: testProjectId, + end_version: 0, + changes: changes.map(changeToRaw), + }) + }) + .then(response => { + expect(response.status).to.equal(HTTPStatus.CREATED) + }) + .then(() => { + return clientForProject.apis.Project.getLatestHistory({ + project_id: testProjectId, + }) + }) + .then(response => { + const chunkResponse = ChunkResponse.fromRaw(response.obj) + const changes = chunkResponse.getChunk().getChanges() + expect(changes.length).to.be.at.least(1) + const lastChange = changes[changes.length - 1] + expect(lastChange.getOrigin()).to.deep.equal(testGitOrigin) + }) + }) + it('rejects text operations on binary files', function () { const testProjectId = '1' const testFilePathname = 'main.tex' @@ -821,9 +885,7 @@ describe('history import', function () { expect.fail() }) .catch(error => { - expect(error.message).to.equal( - 'Required parameter end_version is not provided' - ) + expect(error.message).to.equal('request failed with status 422') }) }) @@ -845,9 +907,6 @@ describe('history import', function () { }) .catch(error => { expect(error.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY) - expect(error.response.body.message).to.equal( - 'invalid enum value: return_snapshot' - ) }) }) }) diff --git a/services/history-v1/test/acceptance/js/api/projects.test.js b/services/history-v1/test/acceptance/js/api/projects.test.js index d72ea66ff0..578e6af94d 100644 --- a/services/history-v1/test/acceptance/js/api/projects.test.js +++ b/services/history-v1/test/acceptance/js/api/projects.test.js @@ -520,7 +520,7 @@ describe('project controller', function () { project_id: projectId, since: -1, }) - ).to.be.rejectedWith('Bad Request') + ).to.be.rejectedWith('request failed with status 400') }) it('rejects out of bounds versions', async function () { @@ -529,7 +529,7 @@ describe('project controller', function () { project_id: projectId, since: 20, }) - ).to.be.rejectedWith('Bad Request') + ).to.be.rejectedWith('request failed with status 400') }) }) diff --git a/services/history-v1/test/acceptance/js/api/support/http_client.js b/services/history-v1/test/acceptance/js/api/support/http_client.js new file mode 100644 index 0000000000..166cb7c0af --- /dev/null +++ b/services/history-v1/test/acceptance/js/api/support/http_client.js @@ -0,0 +1,423 @@ +'use strict' + +const nodeFetch = require('node-fetch') +const { + fetchJsonWithResponse, + fetchStreamWithResponse, + RequestFailedError, +} = require('@overleaf/fetch-utils') + +/** + * Create an HTTP client that mimics the swagger-client API + * @param {string} baseUrl - Base URL for the API + * @param {Object} options - Options including authorizations + * @returns {Object} Client with apis.Project and apis.ProjectImport methods + */ +function createHttpClient(baseUrl, options = {}) { + const authHeaders = {} + const queryParams = {} + + // Handle different auth types + if (options.authorizations) { + if (options.authorizations.jwt) { + if (options.authorizations.jwt.startsWith('Bearer ')) { + authHeaders.Authorization = options.authorizations.jwt + } else if (options.authorizations.jwt.startsWith('Basic ')) { + authHeaders.Authorization = options.authorizations.jwt + } else { + authHeaders.Authorization = `Bearer ${options.authorizations.jwt}` + } + } + if (options.authorizations.basic) { + const { username, password } = options.authorizations.basic + const credentials = Buffer.from(`${username}:${password}`).toString( + 'base64' + ) + authHeaders.Authorization = `Basic ${credentials}` + } + if (options.authorizations.token) { + queryParams.token = options.authorizations.token + } + } + + function makeRequest(method, path, params = {}) { + // Build URL with path params + // Ensure we don't have double slashes between baseUrl and path + const base = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl + const pathStart = path.startsWith('/') ? path : `/${path}` + let url = `${base}${pathStart}` + Object.keys(params).forEach(key => { + if ( + key !== 'body' && + key !== 'since' && + key !== 'end_version' && + key !== 'return_snapshot' && + key !== 'snapshot' && + key !== 'changes' + ) { + const value = params[key] + if (value !== undefined && value !== null) { + url = url.replace(`:${key}`, String(value)) + } + } + }) + + // Build query string + const query = new URLSearchParams() + if (params.since !== undefined) { + query.append('since', params.since) + } + if (params.end_version !== undefined) { + query.append('end_version', params.end_version) + } + if (params.return_snapshot) { + query.append('return_snapshot', params.return_snapshot) + } + Object.keys(queryParams).forEach(key => { + query.append(key, queryParams[key]) + }) + if (query.toString()) { + url += `?${query.toString()}` + } + + // Build headers + const headers = { ...authHeaders } + let body = null + + if (params.body) { + body = JSON.stringify(params.body) + headers['Content-Type'] = 'application/json' + } else if (method === 'POST' || method === 'PUT') { + // Some endpoints have the entire body as the parameter + if (params.snapshot !== undefined) { + body = JSON.stringify(params.snapshot) + headers['Content-Type'] = 'application/json' + } else if (params.changes !== undefined) { + body = JSON.stringify(params.changes) + headers['Content-Type'] = 'application/json' + } + } + + const fetchOpts = { + method, + headers, + body, + } + + const isNoContentEndpoint = + (method === 'POST' && url.includes('/flush')) || + (method === 'POST' && url.includes('/expire')) || + (method === 'DELETE' && + url.includes('/api/projects/') && + !url.includes('/blobs/')) + + const isBlobEndpoint = + method === 'GET' && (url.includes('/blobs/') || url.includes('/zip')) + + if (isNoContentEndpoint) { + return nodeFetch(url, fetchOpts).then(response => { + const headersObj = {} + response.headers.forEach((value, key) => { + headersObj[key.toLowerCase()] = value + }) + + return { + obj: null, + body: null, + status: response.status, + headers: headersObj, + ok: response.ok, + } + }) + } + + if (isBlobEndpoint) { + return fetchStreamWithResponse(url, fetchOpts) + .then(({ stream, response }) => { + const headersObj = {} + response.headers.forEach((value, key) => { + headersObj[key.toLowerCase()] = value + }) + + let cachedBuffer = null + const bufferPromise = (async () => { + if (cachedBuffer === null) { + const chunks = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + cachedBuffer = Buffer.concat(chunks) + } + return cachedBuffer + })() + + return { + obj: null, + status: response.status, + headers: headersObj, + data: { + text: async () => { + const buffer = await bufferPromise + return buffer.toString() + }, + json: async () => { + const buffer = await bufferPromise + return JSON.parse(buffer.toString()) + }, + arrayBuffer: async () => { + const buffer = await bufferPromise + return Uint8Array.from(buffer).buffer + }, + }, + ok: response.ok, + } + }) + .catch(err => { + if (err instanceof RequestFailedError) { + // Convert Headers object to plain object for compatibility with tests + const headersObj = {} + err.response.headers.forEach((value, key) => { + headersObj[key.toLowerCase()] = value + }) + + // Use the server's error message if available, otherwise use the generic message + let errorMessage = err.message + let errorObj = err.body + + if (err.body) { + try { + const bodyObj = + typeof err.body === 'string' ? JSON.parse(err.body) : err.body + errorObj = bodyObj // Store parsed object for error.obj + if (bodyObj?.message) { + errorMessage = bodyObj.message + } else if (typeof bodyObj === 'string' && bodyObj.trim()) { + errorMessage = bodyObj + } + } catch (e) { + // If body isn't JSON, use the string as-is or fall back to generic message + if (typeof err.body === 'string' && err.body.trim()) { + errorMessage = err.body + errorObj = err.body + } + } + } + + // If we still have the generic message, include status code for better debugging + if (errorMessage === 'request failed' && err.response?.status) { + errorMessage = `request failed with status ${err.response.status}` + } + + const error = new Error(errorMessage) + error.status = err.response.status + error.statusCode = err.response.status + error.obj = errorObj + error.response = { + status: err.response.status, + statusCode: err.response.status, + headers: headersObj, // Plain object for err.response.headers access + ok: err.response.ok, + } + throw error + } + throw err + }) + } + + // For JSON endpoints, use fetchJsonWithResponse but handle errors ourselves + return fetchJsonWithResponse(url, fetchOpts) + .then(({ json, response }) => { + // Convert Headers object to plain object for compatibility + const headersObj = {} + response.headers.forEach((value, key) => { + headersObj[key.toLowerCase()] = value + }) + + return { + obj: json, + body: json, + status: response.status, + headers: headersObj, // Plain object for compatibility + ok: response.ok, + } + }) + .catch(err => { + if (err instanceof RequestFailedError) { + // Convert Headers object to plain object for compatibility with tests + const headersObj = {} + err.response.headers.forEach((value, key) => { + headersObj[key.toLowerCase()] = value + }) + + // Preserve the response object structure that tests expect + let errorMessage = err.message + let errorObj = err.body + + // If we still have the generic message, include status code for better debugging + if (errorMessage === 'request failed' && err.response?.status) { + errorMessage = `request failed with status ${err.response.status}` + } + + if (typeof errorObj === 'string') { + try { + errorObj = JSON.parse(errorObj) + } catch (e) { + // leave it as string if it isn't JSON + } + } + + const status = err.response.status + + const error = new Error(errorMessage) + error.status = status + error.statusCode = status + error.obj = errorObj + error.response = { + status, + statusCode: status, + headers: headersObj, // Plain object for err.response.headers['www-authenticate'] access + ok: err.response.ok, + body: errorObj, + } + throw error + } + + // Handle invalid JSON responses (e.g. empty body with 200/204) + if ( + err?.name === 'FetchError' && + (err?.type === 'invalid-json' || + (typeof err.message === 'string' && + err.message.indexOf('invalid json response body') !== -1)) + ) { + const response = err.response + if (response) { + const headersObj = {} + response.headers.forEach((value, key) => { + headersObj[key.toLowerCase()] = value + }) + if (!response.ok) { + const error = new Error( + `request failed with status ${response.status}` + ) + error.status = response.status + error.statusCode = response.status + error.obj = null + error.response = { + status: response.status, + statusCode: response.status, + headers: headersObj, + ok: response.ok, + } + throw error + } + return { + obj: null, + body: null, + status: response.status, + headers: headersObj, + ok: response.ok, + } + } + } + + throw err + }) + } + + return { + apis: { + Project: { + initializeProject: params => + makeRequest('POST', '/api/projects', params), + getProjectBlobsStats: params => + makeRequest('POST', '/api/projects/blob-stats', params), + getBlobStats: params => + makeRequest('POST', `/api/projects/:project_id/blob-stats`, params), + deleteProject: params => + makeRequest('DELETE', `/api/projects/:project_id`, params), + getProjectBlob: params => + makeRequest('GET', `/api/projects/:project_id/blobs/:hash`, params), + headProjectBlob: params => + makeRequest('HEAD', `/api/projects/:project_id/blobs/:hash`, params), + createProjectBlob: params => + makeRequest('PUT', `/api/projects/:project_id/blobs/:hash`, params), + copyProjectBlob: params => + makeRequest('POST', `/api/projects/:project_id/blobs/:hash`, params), + getLatestContent: params => + makeRequest( + 'GET', + `/api/projects/:project_id/latest/content`, + params + ), + getLatestHashedContent: params => + makeRequest( + 'GET', + `/api/projects/:project_id/latest/hashed_content`, + params + ), + getLatestHistory: params => + makeRequest( + 'GET', + `/api/projects/:project_id/latest/history`, + params + ), + getLatestHistoryRaw: params => + makeRequest( + 'GET', + `/api/projects/:project_id/latest/history/raw`, + params + ), + getLatestPersistedHistory: params => + makeRequest( + 'GET', + `/api/projects/:project_id/latest/persistedHistory`, + params + ), + getHistory: params => + makeRequest( + 'GET', + `/api/projects/:project_id/versions/:version/history`, + params + ), + getContentAtVersion: params => + makeRequest( + 'GET', + `/api/projects/:project_id/versions/:version/content`, + params + ), + getHistoryBefore: params => + makeRequest( + 'GET', + `/api/projects/:project_id/timestamp/:timestamp/history`, + params + ), + getZip: params => + makeRequest( + 'GET', + `/api/projects/:project_id/version/:version/zip`, + params + ), + createZip: params => + makeRequest( + 'POST', + `/api/projects/:project_id/version/:version/zip`, + params + ), + getChanges: params => + makeRequest('GET', `/api/projects/:project_id/changes`, params), + }, + ProjectImport: { + importSnapshot1: params => + makeRequest('POST', `/api/projects/:project_id/import`, params), + importChanges1: params => + makeRequest('POST', `/api/projects/:project_id/changes`, params), + flushChanges: params => + makeRequest('POST', `/api/projects/:project_id/flush`, params), + expireProject: params => + makeRequest('POST', `/api/projects/:project_id/expire`, params), + }, + }, + } +} + +module.exports = createHttpClient diff --git a/services/history-v1/test/acceptance/js/api/support/test_server.js b/services/history-v1/test/acceptance/js/api/support/test_server.js index ac6550b5ab..744c8d39f8 100644 --- a/services/history-v1/test/acceptance/js/api/support/test_server.js +++ b/services/history-v1/test/acceptance/js/api/support/test_server.js @@ -11,7 +11,7 @@ const config = require('config') const http = require('node:http') const jwt = require('jsonwebtoken') -const Swagger = require('swagger-client') +const createHttpClient = require('./http_client') const app = require('../../../../../app') @@ -28,10 +28,8 @@ function testUrl(pathname, opts = {}) { exports.url = testUrl function createClient(options) { - // The Swagger client returns native Promises; we use Bluebird promises. Just - // wrapping the client creation is enough in many (but not all) cases to - // get Bluebird into the chain. - return BPromise.resolve(new Swagger(testUrl('/api-docs'), options)) + // Create HTTP client that mimics swagger-client API + return BPromise.resolve(createHttpClient(testUrl(''), options)) } function createTokenForProject(projectId, opts = {}) {