From eddcc5a42e3491217a8fa560161bb8304fb25406 Mon Sep 17 00:00:00 2001 From: Mathias Jakobsen Date: Thu, 7 May 2026 12:09:10 +0100 Subject: [PATCH] Merge pull request #32857 from overleaf/ds-pandoc-import-md [WEB + CLSI] Import markdown files using pandoc GitOrigin-RevId: adad7831ddb13a8fcb8063871166bde13cbbf1b6 --- services/clsi/app.js | 12 +- services/clsi/app/js/ConversionController.js | 16 +- services/clsi/app/js/ConversionManager.js | 42 +- .../test/acceptance/js/ConversionTests.js | 5 +- .../clsi/test/acceptance/js/helpers/Client.js | 6 +- .../test/unit/js/ConversionController.test.js | 100 ++++- .../test/unit/js/ConversionManager.test.js | 416 +++++++++++------- .../Project/ProjectListController.mjs | 1 + .../Uploads/DocumentConversionManager.mjs | 16 +- .../Uploads/ProjectUploadController.mjs | 19 +- .../src/Features/Uploads/UploadsRouter.mjs | 14 +- .../web/frontend/extracted-translations.json | 5 +- .../ide-react/components/global-toasts.tsx | 4 +- .../ide-react/components/modals/modals.tsx | 4 +- ...project-converted-from-document-modal.tsx} | 35 +- .../components/new-project-button.tsx | 18 + .../import-document-feedback-toast.tsx | 67 +++ ...cx-modal.tsx => import-document-modal.tsx} | 44 +- .../import-docx-feedback-toast.tsx | 42 -- .../new-project-button-modal.tsx | 25 +- ...me-message-create-new-project-dropdown.tsx | 17 + .../stylesheets/pages/project-list.scss | 2 +- services/web/locales/en.json | 5 +- .../DocumentConversionManager.test.mjs | 90 +++- .../Uploads/ProjectUploadController.test.mjs | 118 ++++- services/web/types/assets.d.ts | 2 + 26 files changed, 813 insertions(+), 312 deletions(-) rename services/web/frontend/js/features/ide-react/components/modals/{project-converted-from-docx-modal.tsx => project-converted-from-document-modal.tsx} (58%) create mode 100644 services/web/frontend/js/features/project-list/components/new-project-button/import-document-feedback-toast.tsx rename services/web/frontend/js/features/project-list/components/new-project-button/{import-docx-modal.tsx => import-document-modal.tsx} (55%) delete mode 100644 services/web/frontend/js/features/project-list/components/new-project-button/import-docx-feedback-toast.tsx diff --git a/services/clsi/app.js b/services/clsi/app.js index b8d79aaccf..5ea9985a79 100644 --- a/services/clsi/app.js +++ b/services/clsi/app.js @@ -125,10 +125,20 @@ app.get( ) // Conversion endpoints +// Keep old route for backwards compatibility during CLSI/web deploy transition app.post( '/convert/docx-to-latex', FileUploadMiddleware.multerMiddleware, - ConversionController.convertDocxToLaTeX + (req, res, next) => { + req.query.type = 'docx' + next() + }, + ConversionController.convertDocumentToLaTeX +) +app.post( + '/convert/document-to-latex', + FileUploadMiddleware.multerMiddleware, + ConversionController.convertDocumentToLaTeX ) app.post( '/project/:project_id/user/:user_id/download/project-to-document', diff --git a/services/clsi/app/js/ConversionController.js b/services/clsi/app/js/ConversionController.js index eb7f9a79cc..eccdd39ad9 100644 --- a/services/clsi/app/js/ConversionController.js +++ b/services/clsi/app/js/ConversionController.js @@ -11,19 +11,25 @@ import Path from 'node:path' const SUPPORTED_CONVERSION_TYPES = new Map([['docx', 'docx']]) -async function convertDocxToLaTeX(req, res) { +async function convertDocumentToLaTeX(req, res) { const { path } = req.file + const conversionType = req.query.type if (!Settings.enablePandocConversions) { await fs.unlink(path).catch(() => {}) return res.sendStatus(404) } - logger.debug({ path }, 'received file for conversion') + if (!conversionType || !['docx', 'markdown'].includes(conversionType)) { + await fs.unlink(path).catch(() => {}) + return res.sendStatus(400) + } + logger.debug({ path, conversionType }, 'received file for conversion') const conversionId = crypto.randomUUID() let zipPath try { - zipPath = await ConversionManager.promises.convertDocxToLaTeXWithLock( + zipPath = await ConversionManager.promises.convertToLaTeXWithLock( conversionId, - path + path, + conversionType ) } finally { await fs.unlink(path).catch(() => {}) @@ -98,6 +104,6 @@ async function convertProjectToDocument(req, res) { } export default { - convertDocxToLaTeX: expressify(convertDocxToLaTeX), + convertDocumentToLaTeX: expressify(convertDocumentToLaTeX), convertProjectToDocument: expressify(convertProjectToDocument), } diff --git a/services/clsi/app/js/ConversionManager.js b/services/clsi/app/js/ConversionManager.js index 8527064f4c..f8a0a3a049 100644 --- a/services/clsi/app/js/ConversionManager.js +++ b/services/clsi/app/js/ConversionManager.js @@ -6,19 +6,44 @@ import CommandRunner from './CommandRunner.js' import LockManager from './LockManager.js' import OError from '@overleaf/o-error' -async function convertDocxToLaTeXWithLock(conversionId, inputPath) { +const CONVERSION_CONFIGS = { + docx: { + inputFilename: 'input.docx', + pandocArgs: ['--extract-media=.', '--from', 'docx+citations', '--citeproc'], + }, + markdown: { + inputFilename: 'input.md', + pandocArgs: ['--from', 'markdown'], + }, +} + +async function convertToLaTeXWithLock(conversionId, inputPath, conversionType) { const conversionDir = Path.join(Settings.path.compilesDir, conversionId) const lock = LockManager.acquire(conversionDir) try { - return await convertDocxToLaTeX(conversionId, conversionDir, inputPath) + return await convertToLaTeX( + conversionId, + conversionDir, + inputPath, + conversionType + ) } finally { lock.release() } } -async function convertDocxToLaTeX(conversionId, conversionDir, inputPath) { +async function convertToLaTeX( + conversionId, + conversionDir, + inputPath, + conversionType +) { + const config = CONVERSION_CONFIGS[conversionType] + if (!config) { + throw new OError('unsupported conversion type', { conversionType }) + } await fs.mkdir(conversionDir, { recursive: true }) - const newSourcePath = Path.join(conversionDir, 'input.docx') + const newSourcePath = Path.join(conversionDir, config.inputFilename) await fs.copyFile(inputPath, newSourcePath) const outputName = crypto.randomUUID() + '.zip' @@ -31,16 +56,13 @@ async function convertDocxToLaTeX(conversionId, conversionDir, inputPath) { conversionId, [ 'pandoc', - 'input.docx', + config.inputFilename, '--output', 'main.tex', - '--extract-media=.', - '--from', - 'docx+citations', '--to', 'latex', - '--citeproc', '--standalone', + ...config.pandocArgs, ], conversionDir, Settings.pandocImage, @@ -164,7 +186,7 @@ async function convertLaTeXToDocumentInDir( export default { promises: { - convertDocxToLaTeXWithLock, + convertToLaTeXWithLock, convertLaTeXToDocumentInDirWithLock, }, } diff --git a/services/clsi/test/acceptance/js/ConversionTests.js b/services/clsi/test/acceptance/js/ConversionTests.js index 13a733f912..56e93290ec 100644 --- a/services/clsi/test/acceptance/js/ConversionTests.js +++ b/services/clsi/test/acceptance/js/ConversionTests.js @@ -25,7 +25,7 @@ describe('Conversions', function () { const outputStream = fs.createWriteStream( '/tmp/clsi_acceptance_tests_' + crypto.randomUUID() + '.zip' ) - const stream = await Client.convertDocx(sourcePath) + const stream = await Client.convertDocument(sourcePath, 'docx') await pipeline(stream, outputStream) await new Promise((resolve, reject) => { @@ -77,7 +77,8 @@ describe('Conversions', function () { import.meta.dirname, '../fixtures/minimal.pdf' ) - await expect(Client.convertDocx(sourcePath)).to.eventually.be.rejected + await expect(Client.convertDocument(sourcePath, 'docx')).to.eventually.be + .rejected }) }) }) diff --git a/services/clsi/test/acceptance/js/helpers/Client.js b/services/clsi/test/acceptance/js/helpers/Client.js index b7038380c5..fac1d9a26a 100644 --- a/services/clsi/test/acceptance/js/helpers/Client.js +++ b/services/clsi/test/acceptance/js/helpers/Client.js @@ -30,10 +30,10 @@ function compile(projectId, data) { }) } -async function convertDocx(path) { +async function convertDocument(path, type) { const formData = new FormData() formData.append('qqfile', fs.createReadStream(path)) - return await fetchStream(`${host}/convert/docx-to-latex`, { + return await fetchStream(`${host}/convert/document-to-latex?type=${type}`, { method: 'POST', body: formData, }) @@ -202,7 +202,7 @@ function smokeTest() { export default { randomId, compile, - convertDocx, + convertDocument, stopCompile, clearCache, getOutputFile, diff --git a/services/clsi/test/unit/js/ConversionController.test.js b/services/clsi/test/unit/js/ConversionController.test.js index e4f0a0c7f5..849a877646 100644 --- a/services/clsi/test/unit/js/ConversionController.test.js +++ b/services/clsi/test/unit/js/ConversionController.test.js @@ -22,7 +22,7 @@ describe('ConversionController', function () { ctx.parsedRequest = { rootResourcePath: 'main.tex' } ctx.ConversionManager = { promises: { - convertDocxToLaTeXWithLock: sinon.stub().resolves(ctx.zipPath), + convertToLaTeXWithLock: sinon.stub().resolves(ctx.zipPath), convertLaTeXToDocumentInDirWithLock: sinon .stub() .resolves(ctx.documentPath), @@ -86,16 +86,17 @@ describe('ConversionController', function () { ctx.ConversionController = (await import(MODULE_PATH)).default }) - describe('convertDocxToLaTeX', function () { + describe('convertDocumentToLaTeX', function () { describe('when conversions are disabled', function () { beforeEach(async function (ctx) { ctx.Settings.enablePandocConversions = false ctx.req = { file: { path: '/path/to/uploaded/file.docx' }, + query: { type: 'docx' }, } ctx.res.sendStatus = sinon.stub() - await ctx.ConversionController.convertDocxToLaTeX(ctx.req, ctx.res) + await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res) }) it('should remove the uploaded file', function (ctx) { @@ -108,7 +109,59 @@ describe('ConversionController', function () { it('should not call the conversion manager', function (ctx) { sinon.assert.notCalled( - ctx.ConversionManager.promises.convertDocxToLaTeXWithLock + ctx.ConversionManager.promises.convertToLaTeXWithLock + ) + }) + }) + + describe('when conversionType is missing', function () { + beforeEach(async function (ctx) { + ctx.req = { + file: { path: '/path/to/uploaded/file.docx' }, + query: {}, + } + ctx.res.sendStatus = sinon.stub() + + await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res) + }) + + it('should remove the uploaded file', function (ctx) { + sinon.assert.calledWith(ctx.fs.unlink, ctx.req.file.path) + }) + + it('should return 400', function (ctx) { + sinon.assert.calledWith(ctx.res.sendStatus, 400) + }) + + it('should not call the conversion manager', function (ctx) { + sinon.assert.notCalled( + ctx.ConversionManager.promises.convertToLaTeXWithLock + ) + }) + }) + + describe('when conversionType is unsupported', function () { + beforeEach(async function (ctx) { + ctx.req = { + file: { path: '/path/to/uploaded/file.docx' }, + query: { type: 'invalid' }, + } + ctx.res.sendStatus = sinon.stub() + + await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res) + }) + + it('should remove the uploaded file', function (ctx) { + sinon.assert.calledWith(ctx.fs.unlink, ctx.req.file.path) + }) + + it('should return 400', function (ctx) { + sinon.assert.calledWith(ctx.res.sendStatus, 400) + }) + + it('should not call the conversion manager', function (ctx) { + sinon.assert.notCalled( + ctx.ConversionManager.promises.convertToLaTeXWithLock ) }) }) @@ -117,18 +170,20 @@ describe('ConversionController', function () { beforeEach(async function (ctx) { ctx.req = { file: { path: '/path/to/uploaded/file.docx' }, + query: { type: 'docx' }, } - await ctx.ConversionController.convertDocxToLaTeX(ctx.req, ctx.res) + await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res) }) - it('should call the conversion manager with the uploaded file path', function (ctx) { + it('should call the conversion manager with the uploaded file path and type', function (ctx) { sinon.assert.calledWith( - ctx.ConversionManager.promises.convertDocxToLaTeXWithLock, + ctx.ConversionManager.promises.convertToLaTeXWithLock, sinon.match( /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ ), - ctx.req.file.path + ctx.req.file.path, + 'docx' ) }) @@ -160,6 +215,28 @@ describe('ConversionController', function () { }) }) + describe('with conversionType=markdown', function () { + beforeEach(async function (ctx) { + ctx.req = { + file: { path: '/path/to/uploaded/file.md' }, + query: { type: 'markdown' }, + } + + await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res) + }) + + it('should call the conversion manager with the uploaded file path and markdown type', function (ctx) { + sinon.assert.calledWith( + ctx.ConversionManager.promises.convertToLaTeXWithLock, + sinon.match( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ + ), + ctx.req.file.path, + 'markdown' + ) + }) + }) + describe('unsuccessfully', function () { describe('on streaming error', function () { it('should propagate the error and still clean up', async function (ctx) { @@ -169,10 +246,13 @@ describe('ConversionController', function () { res.attachment = sinon.stub() res.setHeader = sinon.stub() - const req = { file: { path: '/path/to/uploaded/file.docx' } } + const req = { + file: { path: '/path/to/uploaded/file.docx' }, + query: { type: 'docx' }, + } await expect( - ctx.ConversionController.convertDocxToLaTeX(req, res) + ctx.ConversionController.convertDocumentToLaTeX(req, res) ).to.be.rejectedWith('mock stream error') sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir) diff --git a/services/clsi/test/unit/js/ConversionManager.test.js b/services/clsi/test/unit/js/ConversionManager.test.js index 24999fccde..49ff520ee8 100644 --- a/services/clsi/test/unit/js/ConversionManager.test.js +++ b/services/clsi/test/unit/js/ConversionManager.test.js @@ -36,7 +36,6 @@ describe('ConversionManager', function () { } ctx.conversionId = 'test-conversion-id' - ctx.inputPath = '/path/to/input.docx' ctx.conversionDir = '/compiles/test-conversion-id' ctx.outputPath = '/compiles/test-conversion-id/output-uuid.zip' @@ -65,188 +64,287 @@ describe('ConversionManager', function () { ctx.uuidStub.restore() }) - describe('convertDocxToLaTeXWithLock', function () { - describe('general behavior', function () { - beforeEach(async function (ctx) { - ctx.result = - await ctx.ConversionManager.promises.convertDocxToLaTeXWithLock( - ctx.conversionId, - ctx.inputPath + describe('convertToLaTeXWithLock', function () { + describe('with conversionType=docx', function () { + beforeEach(function (ctx) { + ctx.inputPath = '/path/to/input.docx' + }) + + describe('file setup and pandoc args', function () { + beforeEach(async function (ctx) { + ctx.result = + await ctx.ConversionManager.promises.convertToLaTeXWithLock( + ctx.conversionId, + ctx.inputPath, + 'docx' + ) + }) + + it('should acquire a lock', async function (ctx) { + sinon.assert.calledWith(ctx.LockManager.acquire, ctx.conversionDir) + }) + + it('should copy the input file to the conversion directory with docx filename', async function (ctx) { + sinon.assert.calledWith(ctx.fs.mkdir, ctx.conversionDir, { + recursive: true, + }) + sinon.assert.calledWith( + ctx.fs.copyFile, + ctx.inputPath, + Path.join(ctx.conversionDir, 'input.docx') ) - }) - - it('should acquire a lock', async function (ctx) { - sinon.assert.calledWith(ctx.LockManager.acquire, ctx.conversionDir) - }) - - it('should copy the input file to the conversion directory', async function (ctx) { - sinon.assert.calledWith(ctx.fs.mkdir, ctx.conversionDir, { - recursive: true, - }) - sinon.assert.calledWith( - ctx.fs.copyFile, - ctx.inputPath, - Path.join(ctx.conversionDir, 'input.docx') - ) - }) - - it('should convert conversion timeout to milliseconds', async function (ctx) { - expect(ctx.CommandRunner.promises.run.firstCall.args[4]).toBe(60_000) - expect(ctx.CommandRunner.promises.run.secondCall.args[4]).toBe(60_000) - }) - - it('should run pandoc followed by zip in the conversion directory', function (ctx) { - expect(ctx.CommandRunner.promises.run.callCount).toBe(2) - expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([ - ctx.conversionId, - [ - 'pandoc', - 'input.docx', - '--output', - 'main.tex', - '--extract-media=.', - '--from', - 'docx+citations', - '--to', - 'latex', - '--citeproc', - '--standalone', - ], - ctx.conversionDir, - ctx.Settings.pandocImage, - 60_000, - {}, - 'conversions', - ]) - expect(ctx.CommandRunner.promises.run.secondCall.args).toEqual([ - ctx.conversionId, - ['zip', '-r', 'output-uuid.zip', '.'], - ctx.conversionDir, - ctx.Settings.pandocImage, - 60_000, - {}, - 'conversions', - ]) - }) - }) - - describe('successful conversion', function () { - beforeEach(async function (ctx) { - ctx.CommandRunner.promises.run.resolves({ - stdout: 'mock-stdout', - stderr: 'mock-stderr', - exitCode: 0, }) - ctx.result = - await ctx.ConversionManager.promises.convertDocxToLaTeXWithLock( + it('should convert conversion timeout to milliseconds', async function (ctx) { + expect(ctx.CommandRunner.promises.run.firstCall.args[4]).toBe(60_000) + expect(ctx.CommandRunner.promises.run.secondCall.args[4]).toBe(60_000) + }) + + it('should run pandoc with docx args followed by zip', function (ctx) { + expect(ctx.CommandRunner.promises.run.callCount).toBe(2) + expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([ ctx.conversionId, - ctx.inputPath - ) - }) - - it('should remove the source document after conversion', async function (ctx) { - sinon.assert.calledWith( - ctx.fs.unlink, - Path.join(ctx.conversionDir, 'input.docx') - ) - }) - - it('should return the conversion directory', function (ctx) { - expect(ctx.result).toBe(ctx.outputPath) - }) - - it('should release the lock', function (ctx) { - sinon.assert.called(ctx.lock.release) - }) - }) - - describe('unsuccessful conversion (exitcode)', function () { - beforeEach(async function (ctx) { - ctx.CommandRunner.promises.run.resolves({ - stdout: 'mock-stdout', - stderr: 'mock-stderr', - exitCode: 63, - }) - - await expect( - ctx.ConversionManager.promises.convertDocxToLaTeXWithLock( + [ + 'pandoc', + 'input.docx', + '--output', + 'main.tex', + '--to', + 'latex', + '--standalone', + '--extract-media=.', + '--from', + 'docx+citations', + '--citeproc', + ], + ctx.conversionDir, + ctx.Settings.pandocImage, + 60_000, + {}, + 'conversions', + ]) + expect(ctx.CommandRunner.promises.run.secondCall.args).toEqual([ ctx.conversionId, - ctx.inputPath - ) - ).to.be.rejectedWith('pandoc conversion failed') - }) - - it('should remove the entire conversion directory', async function (ctx) { - sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, { - force: true, - recursive: true, + ['zip', '-r', 'output-uuid.zip', '.'], + ctx.conversionDir, + ctx.Settings.pandocImage, + 60_000, + {}, + 'conversions', + ]) }) }) - it('should release the lock', function (ctx) { - sinon.assert.called(ctx.lock.release) - }) - }) - - describe('unsuccessful compression (exitcode)', function () { - beforeEach(async function (ctx) { - ctx.CommandRunner.promises.run - .onFirstCall() - .resolves({ - stdout: 'mock-pandoc-stdout', - stderr: 'mock-pandoc-stderr', + describe('successful conversion', function () { + beforeEach(async function (ctx) { + ctx.CommandRunner.promises.run.resolves({ + stdout: 'mock-stdout', + stderr: 'mock-stderr', exitCode: 0, }) - .onSecondCall() - .resolves({ - stdout: 'mock-zip-stdout', - stderr: 'mock-zip-stderr', - exitCode: 12, - }) - await expect( - ctx.ConversionManager.promises.convertDocxToLaTeXWithLock( - ctx.conversionId, - ctx.inputPath + ctx.result = + await ctx.ConversionManager.promises.convertToLaTeXWithLock( + ctx.conversionId, + ctx.inputPath, + 'docx' + ) + }) + + it('should remove the source document after conversion', async function (ctx) { + sinon.assert.calledWith( + ctx.fs.unlink, + Path.join(ctx.conversionDir, 'input.docx') ) - ).to.be.rejectedWith('pandoc conversion failed') - }) + }) - it('should remove the entire conversion directory', async function (ctx) { - sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, { - force: true, - recursive: true, + it('should return the output zip path', function (ctx) { + expect(ctx.result).toBe(ctx.outputPath) + }) + + it('should release the lock', function (ctx) { + sinon.assert.called(ctx.lock.release) }) }) - it('should release the lock', function (ctx) { - sinon.assert.called(ctx.lock.release) + describe('unsuccessful conversion (exitcode)', function () { + beforeEach(async function (ctx) { + ctx.CommandRunner.promises.run.resolves({ + stdout: 'mock-stdout', + stderr: 'mock-stderr', + exitCode: 63, + }) + + await expect( + ctx.ConversionManager.promises.convertToLaTeXWithLock( + ctx.conversionId, + ctx.inputPath, + 'docx' + ) + ).to.be.rejectedWith('pandoc conversion failed') + }) + + it('should remove the entire conversion directory', async function (ctx) { + sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, { + force: true, + recursive: true, + }) + }) + + it('should release the lock', function (ctx) { + sinon.assert.called(ctx.lock.release) + }) + }) + + describe('unsuccessful compression (exitcode)', function () { + beforeEach(async function (ctx) { + ctx.CommandRunner.promises.run + .onFirstCall() + .resolves({ + stdout: 'mock-pandoc-stdout', + stderr: 'mock-pandoc-stderr', + exitCode: 0, + }) + .onSecondCall() + .resolves({ + stdout: 'mock-zip-stdout', + stderr: 'mock-zip-stderr', + exitCode: 12, + }) + + await expect( + ctx.ConversionManager.promises.convertToLaTeXWithLock( + ctx.conversionId, + ctx.inputPath, + 'docx' + ) + ).to.be.rejectedWith('pandoc conversion failed') + }) + + it('should remove the entire conversion directory', async function (ctx) { + sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, { + force: true, + recursive: true, + }) + }) + + it('should release the lock', function (ctx) { + sinon.assert.called(ctx.lock.release) + }) + }) + + describe('unsuccessful conversion (throws)', function () { + beforeEach(async function (ctx) { + ctx.CommandRunner.promises.run.rejects( + new Error('mock conversion error') + ) + await expect( + ctx.ConversionManager.promises.convertToLaTeXWithLock( + ctx.conversionId, + ctx.inputPath, + 'docx' + ) + ).to.be.rejectedWith('pandoc conversion failed') + }) + + it('should remove the entire conversion directory', async function (ctx) { + sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, { + force: true, + recursive: true, + }) + }) + + it('should release the lock', function (ctx) { + sinon.assert.called(ctx.lock.release) + }) }) }) - describe('unsuccessful conversion (throws)', function () { - beforeEach(async function (ctx) { - ctx.CommandRunner.promises.run.rejects( - new Error('mock conversion error') - ) - await expect( - ctx.ConversionManager.promises.convertDocxToLaTeXWithLock( - ctx.conversionId, - ctx.inputPath - ) - ).to.be.rejectedWith('pandoc conversion failed') + describe('with conversionType=markdown', function () { + beforeEach(function (ctx) { + ctx.inputPath = '/path/to/input.md' }) - it('should remove the entire conversion directory', async function (ctx) { - sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, { - force: true, - recursive: true, + describe('file setup and pandoc args', function () { + beforeEach(async function (ctx) { + ctx.result = + await ctx.ConversionManager.promises.convertToLaTeXWithLock( + ctx.conversionId, + ctx.inputPath, + 'markdown' + ) + }) + + it('should copy the input file to the conversion directory with md filename', async function (ctx) { + sinon.assert.calledWith(ctx.fs.mkdir, ctx.conversionDir, { + recursive: true, + }) + sinon.assert.calledWith( + ctx.fs.copyFile, + ctx.inputPath, + Path.join(ctx.conversionDir, 'input.md') + ) + }) + + it('should run pandoc with markdown args followed by zip', function (ctx) { + expect(ctx.CommandRunner.promises.run.callCount).toBe(2) + expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([ + ctx.conversionId, + [ + 'pandoc', + 'input.md', + '--output', + 'main.tex', + '--to', + 'latex', + '--standalone', + '--from', + 'markdown', + ], + ctx.conversionDir, + ctx.Settings.pandocImage, + 60_000, + {}, + 'conversions', + ]) + expect(ctx.CommandRunner.promises.run.secondCall.args).toEqual([ + ctx.conversionId, + ['zip', '-r', 'output-uuid.zip', '.'], + ctx.conversionDir, + ctx.Settings.pandocImage, + 60_000, + {}, + 'conversions', + ]) }) }) - it('should release the lock', function (ctx) { - sinon.assert.called(ctx.lock.release) + describe('successful conversion', function () { + beforeEach(async function (ctx) { + ctx.CommandRunner.promises.run.resolves({ + stdout: 'mock-stdout', + stderr: 'mock-stderr', + exitCode: 0, + }) + + ctx.result = + await ctx.ConversionManager.promises.convertToLaTeXWithLock( + ctx.conversionId, + ctx.inputPath, + 'markdown' + ) + }) + + it('should remove the source document after conversion', async function (ctx) { + sinon.assert.calledWith( + ctx.fs.unlink, + Path.join(ctx.conversionDir, 'input.md') + ) + }) + + it('should return the output zip path', function (ctx) { + expect(ctx.result).toBe(ctx.outputPath) + }) }) }) }) diff --git a/services/web/app/src/Features/Project/ProjectListController.mjs b/services/web/app/src/Features/Project/ProjectListController.mjs index 9f174f31af..9a6864278e 100644 --- a/services/web/app/src/Features/Project/ProjectListController.mjs +++ b/services/web/app/src/Features/Project/ProjectListController.mjs @@ -531,6 +531,7 @@ async function projectListPage(req, res, next) { // Split tests that will be made available to the frontend 'import-docx', 'overleaf-library', + 'import-markdown', ].filter(Boolean) await Promise.all( diff --git a/services/web/app/src/Features/Uploads/DocumentConversionManager.mjs b/services/web/app/src/Features/Uploads/DocumentConversionManager.mjs index a4b9fe6887..5654f12830 100644 --- a/services/web/app/src/Features/Uploads/DocumentConversionManager.mjs +++ b/services/web/app/src/Features/Uploads/DocumentConversionManager.mjs @@ -11,20 +11,26 @@ import OError from '@overleaf/o-error' import FormData from 'form-data' import { FileTooLargeError } from '../Errors/Errors.js' -async function convertDocxToLaTeXZipArchive(path, userId) { +async function convertDocumentToLaTeXZipArchive(path, userId, conversionType) { const clsiUrl = new URL(Settings.apis.clsi.url) const limits = await CompileManager.promises._getUserCompileLimits(userId) - clsiUrl.pathname = '/convert/docx-to-latex' + // Uncomment this and remove the line below when the deploy is done. + // clsiUrl.pathname = '/convert/document-to-latex' + clsiUrl.pathname = + conversionType === 'docx' + ? '/convert/docx-to-latex' + : '/convert/document-to-latex' clsiUrl.searchParams.set('compileBackendClass', limits.compileBackendClass) clsiUrl.searchParams.set('compileGroup', limits.compileGroup) + clsiUrl.searchParams.set('type', conversionType) const formData = new FormData() formData.append('qqfile', fs.createReadStream(path)) logger.debug( - { clsiUrl: clsiUrl.toString() }, - 'sending docx to CLSI for conversion' + { clsiUrl: clsiUrl.toString(), conversionType }, + 'sending document to CLSI for conversion' ) const outputFileName = crypto.randomUUID() + '_document-conversion' + '.zip' @@ -99,7 +105,7 @@ async function convertProjectToDocument(projectId, userId, type) { export default { promises: { - convertDocxToLaTeXZipArchive, + convertDocumentToLaTeXZipArchive, convertProjectToDocument, }, } diff --git a/services/web/app/src/Features/Uploads/ProjectUploadController.mjs b/services/web/app/src/Features/Uploads/ProjectUploadController.mjs index bdc703aa15..22a88a477a 100644 --- a/services/web/app/src/Features/Uploads/ProjectUploadController.mjs +++ b/services/web/app/src/Features/Uploads/ProjectUploadController.mjs @@ -178,16 +178,21 @@ async function uploadFile(req, res, next) { * @param {any} res * @param {any} next */ -async function importDocx(req, res, next) { +async function importDocument(req, res, next) { const userId = SessionManager.getLoggedInUserId(req.session) - logger.debug({ path: req.file?.path, userId }, 'importing docx file') const { path } = req.file - const name = Path.basename(req.body.name, '.docx') + const conversionType = req.query.type + if (!['docx', 'markdown'].includes(conversionType)) { + return res.status(400).json({ success: false, error: 'invalid_type' }) + } + const name = Path.basename(req.body.name, Path.extname(req.body.name)) + logger.debug({ path, userId, conversionType }, 'importing document file') try { const archivePath = - await DocumentConversionManager.promises.convertDocxToLaTeXZipArchive( + await DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive( path, - userId + userId, + conversionType ) try { const project = @@ -207,7 +212,7 @@ async function importDocx(req, res, next) { }) } } catch (error) { - logger.error({ error }, 'error importing docx file') + logger.error({ error }, 'error importing document file') if ( error instanceof FileTooLargeError || error?.name === 'FileTooLargeError' @@ -267,5 +272,5 @@ export default { uploadProject, uploadFile: expressify(uploadFile), multerMiddleware, - importDocx: expressify(importDocx), + importDocument: expressify(importDocument), } diff --git a/services/web/app/src/Features/Uploads/UploadsRouter.mjs b/services/web/app/src/Features/Uploads/UploadsRouter.mjs index 98716f2ba1..070cc90801 100644 --- a/services/web/app/src/Features/Uploads/UploadsRouter.mjs +++ b/services/web/app/src/Features/Uploads/UploadsRouter.mjs @@ -28,12 +28,24 @@ export default { ) if (Settings.enablePandocConversions) { + webRouter.post( + '/project/new/import-document', + AuthenticationController.requireLogin(), + RateLimiterMiddleware.rateLimit(rateLimiters.projectUpload), + ProjectUploadController.multerMiddleware, + ProjectUploadController.importDocument + ) + // Keep old route for backwards compatibility with old frontends that haven't reloaded webRouter.post( '/project/new/import-docx', AuthenticationController.requireLogin(), RateLimiterMiddleware.rateLimit(rateLimiters.projectUpload), ProjectUploadController.multerMiddleware, - ProjectUploadController.importDocx + (req, res, next) => { + req.query.type = 'docx' + next() + }, + ProjectUploadController.importDocument ) } diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index 4f090bbe3b..535c5c8728 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -287,6 +287,7 @@ "choose_a_custom_color": "", "choose_from_group_members": "", "choose_how_you_search_your_references": "", + "choose_markdown_file": "", "choose_which_experiments": "", "choose_word_document": "", "citation": "", @@ -905,12 +906,13 @@ "image_url": "", "image_width": "", "import_a_bibtex_file_from_your_provider_account": "", + "import_document_description": "", "import_existing_projects_from_github": "", "import_from_github": "", "import_idp_metadata": "", + "import_markdown_file": "", "import_to_sharelatex": "", "import_word_document": "", - "import_word_document_description": "", "imported_from_another_project_at_date": "", "imported_from_external_provider_at_date": "", "imported_from_mendeley_at_date": "", @@ -1148,6 +1150,7 @@ "manager": "", "managers_management": "", "managing_your_subscription": "", + "markdown_import_feedback_message": "", "marked_as_resolved": "", "math": "", "math_display": "", diff --git a/services/web/frontend/js/features/ide-react/components/global-toasts.tsx b/services/web/frontend/js/features/ide-react/components/global-toasts.tsx index e2faf57d83..938dc7a662 100644 --- a/services/web/frontend/js/features/ide-react/components/global-toasts.tsx +++ b/services/web/frontend/js/features/ide-react/components/global-toasts.tsx @@ -6,7 +6,7 @@ import { debugConsole } from '@/utils/debugging' import importOverleafModules from '../../../../macros/import-overleaf-module.macro' import { OLToastContainer } from '@/shared/components/ol/ol-toast-container' import clipboardToastGenerators from '@/features/source-editor/components/clipboard-toasts' -import importDocxFeedbackToastGenerators from '@/features/project-list/components/new-project-button/import-docx-feedback-toast' +import importDocumentFeedbackToastGenerators from '@/features/project-list/components/new-project-button/import-document-feedback-toast' import exportDocumentToastGenerators from '@/features/ide-react/components/toolbar/export-document-toasts' const moduleGeneratorsImport = importOverleafModules('toastGenerators') as { @@ -29,7 +29,7 @@ type GlobalToastGenerator = ( const GENERATOR_LIST: GlobalToastGeneratorEntry[] = [ ...moduleGenerators.flat(), ...clipboardToastGenerators, - ...importDocxFeedbackToastGenerators, + ...importDocumentFeedbackToastGenerators, ...exportDocumentToastGenerators, ] const GENERATOR_MAP: Map = new Map( diff --git a/services/web/frontend/js/features/ide-react/components/modals/modals.tsx b/services/web/frontend/js/features/ide-react/components/modals/modals.tsx index 15820f4fbe..017d03646a 100644 --- a/services/web/frontend/js/features/ide-react/components/modals/modals.tsx +++ b/services/web/frontend/js/features/ide-react/components/modals/modals.tsx @@ -3,7 +3,7 @@ import ForceDisconnected from '@/features/ide-react/components/modals/force-disc import { UnsavedDocs } from '@/features/ide-react/components/unsaved-docs/unsaved-docs' import SystemMessages from '@/shared/components/system-messages' import ViewOnlyAccessModal from '@/features/share-project-modal/components/view-only-access-modal' -import ProjectConvertedFromDocxModal from '@/features/ide-react/components/modals/project-converted-from-docx-modal' +import ProjectConvertedFromDocumentModal from '@/features/ide-react/components/modals/project-converted-from-document-modal' export const Modals = memo(() => { return ( @@ -12,7 +12,7 @@ export const Modals = memo(() => { - + ) }) diff --git a/services/web/frontend/js/features/ide-react/components/modals/project-converted-from-docx-modal.tsx b/services/web/frontend/js/features/ide-react/components/modals/project-converted-from-document-modal.tsx similarity index 58% rename from services/web/frontend/js/features/ide-react/components/modals/project-converted-from-docx-modal.tsx rename to services/web/frontend/js/features/ide-react/components/modals/project-converted-from-document-modal.tsx index 193fb93362..b013094b0f 100644 --- a/services/web/frontend/js/features/ide-react/components/modals/project-converted-from-docx-modal.tsx +++ b/services/web/frontend/js/features/ide-react/components/modals/project-converted-from-document-modal.tsx @@ -8,36 +8,35 @@ import { } from '@/shared/components/ol/ol-modal' import OLButton from '@/shared/components/ol/ol-button' import { useEffect, useState } from 'react' -import { showImportDocxFeedbackToast } from '@/features/project-list/components/new-project-button/import-docx-feedback-toast' +import { showImportDocumentFeedbackToast } from '@/features/project-list/components/new-project-button/import-document-feedback-toast' -function ProjectConvertedFromDocxModal() { - const [ - showProjectConvertedFromDocxModal, - setShowProjectConvertedFromDocxModal, - ] = useState(false) +function ProjectConvertedFromDocumentModal() { + const [convertedFrom, setConvertedFrom] = useState(null) useEffect(() => { - const query = window.location.search - const queryString = new URLSearchParams(query) + const queryString = new URLSearchParams(window.location.search) + const from = queryString.get('converted-from') - if (queryString.get('converted-from-docx') === 'true') { - setShowProjectConvertedFromDocxModal(true) + if (from) { + setConvertedFrom(from) // Clean the URL immediately so a refresh doesn't trigger the modal again, // but preserve other search params and the hash. const url = new URL(window.location.href) - url.searchParams.delete('converted-from-docx') + url.searchParams.delete('converted-from') window.history.replaceState(window.history.state, '', url.toString()) } }, []) return ( <> - {showProjectConvertedFromDocxModal && ( - { - setShowProjectConvertedFromDocxModal(false) - showImportDocxFeedbackToast() + setConvertedFrom(null) + if (convertedFrom === 'docx' || convertedFrom === 'markdown') { + showImportDocumentFeedbackToast(convertedFrom) + } }} /> )} @@ -45,7 +44,7 @@ function ProjectConvertedFromDocxModal() { ) } -function ProjectConvertedFromDocxModalContent({ +function ProjectConvertedFromImportModalContent({ onHide, }: { onHide: () => void @@ -57,7 +56,7 @@ function ProjectConvertedFromDocxModalContent({ show animation onHide={onHide} - id="converted-from-docx-modal" + id="converted-from-document-modal" backdrop="static" > @@ -73,4 +72,4 @@ function ProjectConvertedFromDocxModalContent({ ) } -export default ProjectConvertedFromDocxModal +export default ProjectConvertedFromDocumentModal diff --git a/services/web/frontend/js/features/project-list/components/new-project-button.tsx b/services/web/frontend/js/features/project-list/components/new-project-button.tsx index ed36d93a26..86731f1df7 100644 --- a/services/web/frontend/js/features/project-list/components/new-project-button.tsx +++ b/services/web/frontend/js/features/project-list/components/new-project-button.tsx @@ -62,6 +62,9 @@ function NewProjectButton({ const docxImportEnabled = useFeatureFlag('import-docx') && getMeta('ol-ExposedSettings').enablePandocConversions + const markdownImportEnabled = + useFeatureFlag('import-markdown') && + getMeta('ol-ExposedSettings').enablePandocConversions const sendTrackingEvent = useCallback( ({ dropdownMenu, @@ -228,6 +231,21 @@ function NewProjectButton({ )} + {markdownImportEnabled && ( +
  • + + handleModalMenuClick(e, { + modalVariant: 'import_markdown', + dropdownMenuEvent: 'import-markdown', + }) + } + trailingIcon={} + > + {t('import_markdown_file')} + +
    • + )}
  • {ImportProjectFromGithubMenu && ( ( +
    + , + ]} + /> +
    +) + +const MarkdownImportFeedbackToast = () => ( +
    + , + ]} + /> +
    +) + +const generators: GlobalToastGeneratorEntry[] = [ + { + key: 'import:docx-feedback', + generator: () => ({ + content: , + type: 'info', + autoHide: false, + isDismissible: true, + }), + }, + { + key: 'import:markdown-feedback', + generator: () => ({ + content: , + type: 'info', + autoHide: false, + isDismissible: true, + }), + }, +] + +export default generators + +export const showImportDocumentFeedbackToast = (type: 'docx' | 'markdown') => { + const key = + type === 'markdown' ? 'import:markdown-feedback' : 'import:docx-feedback' + window.dispatchEvent( + new CustomEvent('ide:show-toast', { + detail: { key }, + }) + ) +} diff --git a/services/web/frontend/js/features/project-list/components/new-project-button/import-docx-modal.tsx b/services/web/frontend/js/features/project-list/components/new-project-button/import-document-modal.tsx similarity index 55% rename from services/web/frontend/js/features/project-list/components/new-project-button/import-docx-modal.tsx rename to services/web/frontend/js/features/project-list/components/new-project-button/import-document-modal.tsx index 740a1c2cd9..e23c0c1c5a 100644 --- a/services/web/frontend/js/features/project-list/components/new-project-button/import-docx-modal.tsx +++ b/services/web/frontend/js/features/project-list/components/new-project-button/import-document-modal.tsx @@ -1,3 +1,4 @@ +import { useMemo } from 'react' import { Dashboard } from '@uppy/react' import { useTranslation } from 'react-i18next' import { useProjectUploader } from '../../hooks/use-project-uploader' @@ -13,19 +14,39 @@ import '@uppy/core/dist/style.css' import '@uppy/dashboard/dist/style.css' import BetaBadgeIcon from '@/shared/components/beta-badge-icon' -function ImportDocxModal({ +function ImportDocumentModal({ + type, onHide, openProject, }: { + type: 'docx' | 'markdown' onHide: () => void - openProject: (id: string, isConvertedFromDocx?: boolean) => void + openProject: (id: string, convertedFrom?: string) => void }) { const { t } = useTranslation() + const IMPORT_CONFIGS = useMemo( + () => ({ + docx: { + allowedFileTypes: ['.docx'], + title: t('choose_word_document'), + browseLabel: 'Select .docx file', + dragLabel: '%{browseFiles} or \n\n Drag .docx file', + }, + markdown: { + allowedFileTypes: ['.md'], + title: t('choose_markdown_file'), + browseLabel: 'Select .md file', + dragLabel: '%{browseFiles} or \n\n Drag .md file', + }, + }), + [t] + ) + const config = IMPORT_CONFIGS[type] const uppy = useProjectUploader({ - endpoint: '/project/new/import-docx', - allowedFileTypes: ['.docx'], - onSuccess: (projectId: string) => openProject(projectId, true), + endpoint: `/project/new/import-document?type=${type}`, + allowedFileTypes: config.allowedFileTypes, + onSuccess: (projectId: string) => openProject(projectId, type), }) return ( @@ -36,16 +57,17 @@ function ImportDocxModal({ id="upload-project-modal" backdrop="static" > + {/* TODO: make necessary changes here for import document modal */} - - {t('choose_word_document')} + + {config.title} -

    {t('import_word_document_description')}

    +

    {t('import_document_description')}

    { - return ( -
    - , - ]} - /> -
    - ) -} - -const generators: GlobalToastGeneratorEntry[] = [ - { - key: 'import:docx-feedback', - generator: () => ({ - content: , - type: 'info', - autoHide: false, - isDismissible: true, - }), - }, -] - -export default generators - -export const showImportDocxFeedbackToast = () => { - window.dispatchEvent( - new CustomEvent('ide:show-toast', { - detail: { key: 'import:docx-feedback' }, - }) - ) -} diff --git a/services/web/frontend/js/features/project-list/components/new-project-button/new-project-button-modal.tsx b/services/web/frontend/js/features/project-list/components/new-project-button/new-project-button-modal.tsx index d8adc70ca0..bfac2356c1 100644 --- a/services/web/frontend/js/features/project-list/components/new-project-button/new-project-button-modal.tsx +++ b/services/web/frontend/js/features/project-list/components/new-project-button/new-project-button-modal.tsx @@ -7,7 +7,7 @@ import { FullSizeLoadingSpinner } from '@/shared/components/loading-spinner' import { useLocation } from '@/shared/hooks/use-location' const UploadProjectModal = lazy(() => import('./upload-project-modal')) -const ImportDocxModal = lazy(() => import('./import-docx-modal')) +const ImportDocumentModal = lazy(() => import('./import-document-modal')) export type NewProjectButtonModalVariant = | 'blank_project' @@ -15,6 +15,7 @@ export type NewProjectButtonModalVariant = | 'upload_project' | 'import_from_github' | 'import_docx' + | 'import_markdown' type NewProjectButtonModalProps = { modal: Nullable @@ -32,9 +33,9 @@ function NewProjectButtonModal({ modal, onHide }: NewProjectButtonModalProps) { const location = useLocation() const openProject = useCallback( - (projectId: string, isConvertedFromDocx: boolean = false) => { - const url = isConvertedFromDocx - ? `/project/${projectId}?converted-from-docx=true` + (projectId: string, convertedFrom?: string) => { + const url = convertedFrom + ? `/project/${projectId}?converted-from=${convertedFrom}` : `/project/${projectId}` location.assign(url) @@ -56,7 +57,21 @@ function NewProjectButtonModal({ modal, onHide }: NewProjectButtonModalProps) { case 'import_docx': return ( }> - + + + ) + case 'import_markdown': + return ( + }> + ) case 'import_from_github': diff --git a/services/web/frontend/js/features/project-list/components/welcome-message-new/welcome-message-create-new-project-dropdown.tsx b/services/web/frontend/js/features/project-list/components/welcome-message-new/welcome-message-create-new-project-dropdown.tsx index ec697d2315..7c3b0e9573 100644 --- a/services/web/frontend/js/features/project-list/components/welcome-message-new/welcome-message-create-new-project-dropdown.tsx +++ b/services/web/frontend/js/features/project-list/components/welcome-message-new/welcome-message-create-new-project-dropdown.tsx @@ -64,6 +64,9 @@ function WelcomeMessageCreateNewProjectDropdown({ const docxImportEnabled = useFeatureFlag('import-docx') && getMeta('ol-ExposedSettings').enablePandocConversions + const markdownImportEnabled = + useFeatureFlag('import-markdown') && + getMeta('ol-ExposedSettings').enablePandocConversions const { isOverleaf } = getMeta('ol-ExposedSettings') @@ -153,6 +156,20 @@ function WelcomeMessageCreateNewProjectDropdown({
    • )} + {markdownImportEnabled && ( +
  • + + handleDropdownItemClick(e, 'import_markdown', 'import-markdown') + } + tabIndex={-1} + trailingIcon={} + > + {t('import_markdown_file')} + +
    • + )} {isOverleaf && (
  • Another project/__sourceEntityPathHTML__, at __formattedDate__ __relativeDate__", "imported_from_external_provider_at_date": "Imported from <0>__shortenedUrlHTML__ at __formattedDate__ __relativeDate__", @@ -1517,6 +1519,7 @@ "managers_management": "Managers management", "managing_your_subscription": "Managing your subscription", "march": "March", + "markdown_import_feedback_message": "Importing Markdown files is a new feature. <0>Let us know what you think", "marked_as_resolved": "Marked as resolved", "math": "Math", "math_display": "Math Display", diff --git a/services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs b/services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs index 827dbd6d31..2296ddcbe7 100644 --- a/services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs +++ b/services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs @@ -88,12 +88,74 @@ describe('DocumentConversionManager', function () { ctx.DocumentConversionManager = (await import(MODULE_PATH)).default }) - describe('convertDocxToLaTeXZipArchive', function () { - describe('successfully', function () { + describe('convertDocumentToLaTeXZipArchive', function () { + describe('with conversionType=docx', function () { + describe('successfully', function () { + beforeEach(async function (ctx) { + ctx.path = '/path/to/input.docx' + ctx.userId = 'test-user-id' + ctx.response = { + headers: { + get: sinon.stub().returns(null), + }, + } + ctx.response.headers.get.withArgs('Content-Length').returns('50') + + ctx.fetchUtils.fetchStreamWithResponse.resolves({ + stream: 'mocked-fetch-stream', + response: ctx.response, + }) + + ctx.result = + await ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive( + ctx.path, + ctx.userId, + 'docx' + ) + }) + + it('should call fetchStreamWithResponse with the correct URL and form data', function (ctx) { + const expectedUrl = new URL(ctx.Settings.apis.clsi.url) + // TODO: revert this to '/convert/document-to-latex' once the deploy is done (PR #32857) + expectedUrl.pathname = '/convert/docx-to-latex' + expectedUrl.searchParams.set( + 'compileBackendClass', + 'test-backend-class' + ) + expectedUrl.searchParams.set('compileGroup', 'test-compile-group') + expectedUrl.searchParams.set('type', 'docx') + + sinon.assert.calledWith( + ctx.fetchUtils.fetchStreamWithResponse, + sinon.match(url => url.toString() === expectedUrl.toString()), + { + method: 'POST', + body: sinon.match.instanceOf(FormData), + signal: sinon.match.instanceOf(AbortSignal), + } + ) + }) + + it('should pipe result into the output file', function (ctx) { + sinon.assert.calledWith( + ctx.nodeStream.pipeline, + 'mocked-fetch-stream', + 'mocked-write-stream' + ) + }) + + it('should return a path to the output file', function (ctx) { + expect(ctx.result).to.match( + /\/path\/to\/dump\/folder\/[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}_document-conversion\.zip/ + ) + }) + }) + }) + + describe('with conversionType=markdown', function () { beforeEach(async function (ctx) { - ctx.path = '/path/to/input.docx' + ctx.path = '/path/to/input.md' ctx.userId = 'test-user-id' - ctx.outputPath = '/path/to/output.zip' ctx.response = { headers: { get: sinon.stub().returns(null), @@ -107,20 +169,22 @@ describe('DocumentConversionManager', function () { }) ctx.result = - await ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive( + await ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive( ctx.path, - ctx.userId + ctx.userId, + 'markdown' ) }) - it('should call fetchStreamWithResponse with the correct URL and form data', function (ctx) { + it('should call fetchStreamWithResponse with the correct URL including markdown type', function (ctx) { const expectedUrl = new URL(ctx.Settings.apis.clsi.url) - expectedUrl.pathname = '/convert/docx-to-latex' + expectedUrl.pathname = '/convert/document-to-latex' expectedUrl.searchParams.set( 'compileBackendClass', 'test-backend-class' ) expectedUrl.searchParams.set('compileGroup', 'test-compile-group') + expectedUrl.searchParams.set('type', 'markdown') sinon.assert.calledWith( ctx.fetchUtils.fetchStreamWithResponse, @@ -158,9 +222,10 @@ describe('DocumentConversionManager', function () { ) await expect( - ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive( + ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive( ctx.path, - ctx.userId + ctx.userId, + 'docx' ) ).to.be.rejectedWith('document conversion failed') }) @@ -195,9 +260,10 @@ describe('DocumentConversionManager', function () { }) await expect( - ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive( + ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive( ctx.path, - ctx.userId + ctx.userId, + 'docx' ) ).to.be.rejectedWith(sinon.match.instanceOf(FileTooLargeError)) }) diff --git a/services/web/test/unit/src/Uploads/ProjectUploadController.test.mjs b/services/web/test/unit/src/Uploads/ProjectUploadController.test.mjs index b6c6c75580..1c2fc5476b 100644 --- a/services/web/test/unit/src/Uploads/ProjectUploadController.test.mjs +++ b/services/web/test/unit/src/Uploads/ProjectUploadController.test.mjs @@ -48,7 +48,7 @@ describe('ProjectUploadController', function () { } ctx.DocumentConversionManager = { promises: { - convertDocxToLaTeXZipArchive: sinon.stub(), + convertDocumentToLaTeXZipArchive: sinon.stub(), }, } @@ -463,7 +463,7 @@ describe('ProjectUploadController', function () { }) }) - describe('importDocx', function () { + describe('importDocument', function () { beforeEach(async function (ctx) { ctx.req.file = { path: '/path/to/uploaded/file.docx', @@ -471,13 +471,73 @@ describe('ProjectUploadController', function () { ctx.req.body = { name: 'file.docx', } + ctx.req.query = { type: 'docx' } ctx.archivePath = '/path/to/archive.zip' ctx.fsPromises.unlink = sinon.stub().resolves() }) - describe('successfully', async function () { + describe('with conversionType=docx', async function () { + describe('successfully', async function () { + beforeEach(async function (ctx) { + ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive = + sinon.stub().resolves(ctx.archivePath) + ctx.ProjectUploadManager.promises.createProjectFromZipArchive = sinon + .stub() + .resolves({ + _id: 'new-project-id', + }) + + await new Promise(resolve => { + ctx.res.json = data => { + expect(data.success).to.be.true + expect(data.project_id).to.equal('new-project-id') + resolve() + } + ctx.ProjectUploadController.importDocument(ctx.req, ctx.res) + }) + }) + + it('should call the DocumentConversionManager with file path and type', function (ctx) { + expect( + ctx.DocumentConversionManager.promises + .convertDocumentToLaTeXZipArchive + ).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'docx') + }) + + it('should use the resulting archive to create a new project', function (ctx) { + expect( + ctx.ProjectUploadManager.promises.createProjectFromZipArchive + ).to.have.been.calledWith(ctx.user_id, 'file', ctx.archivePath) + }) + + it('should set the compiler to lualatex', function (ctx) { + expect( + ctx.ProjectOptionsHandler.promises.setCompiler + ).to.have.been.calledWith('new-project-id', 'lualatex') + }) + + it('should unlink the archive after creating the project', function (ctx) { + expect(ctx.fsPromises.unlink).to.have.been.calledWith(ctx.archivePath) + }) + + it('should unlink the uploaded file', function (ctx) { + expect(ctx.fsPromises.unlink).to.have.been.calledWith( + ctx.req.file.path + ) + }) + }) + }) + + describe('with conversionType=markdown', async function () { beforeEach(async function (ctx) { - ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive = + ctx.req.file = { + path: '/path/to/uploaded/file.md', + } + ctx.req.body = { + name: 'file.md', + } + ctx.req.query = { type: 'markdown' } + ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive = sinon.stub().resolves(ctx.archivePath) ctx.ProjectUploadManager.promises.createProjectFromZipArchive = sinon .stub() @@ -491,14 +551,15 @@ describe('ProjectUploadController', function () { expect(data.project_id).to.equal('new-project-id') resolve() } - ctx.ProjectUploadController.importDocx(ctx.req, ctx.res) + ctx.ProjectUploadController.importDocument(ctx.req, ctx.res) }) }) - it('should call the DocumentConversionManager to convert the file', function (ctx) { + it('should call the DocumentConversionManager with file path and markdown type', function (ctx) { expect( - ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive - ).to.have.been.calledWith(ctx.req.file.path, ctx.user_id) + ctx.DocumentConversionManager.promises + .convertDocumentToLaTeXZipArchive + ).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'markdown') }) it('should use the resulting archive to create a new project', function (ctx) { @@ -522,9 +583,37 @@ describe('ProjectUploadController', function () { }) }) + describe('with an invalid conversionType', async function () { + beforeEach(async function (ctx) { + ctx.req.query = { type: 'invalid' } + + await new Promise(resolve => { + ctx.res.json = data => { + expect(data).to.deep.equal({ + success: false, + error: 'invalid_type', + }) + resolve() + } + ctx.ProjectUploadController.importDocument(ctx.req, ctx.res) + }) + }) + + it('should return http 400', function (ctx) { + expect(ctx.res.statusCode).to.equal(400) + }) + + it('should not call DocumentConversionManager', function (ctx) { + expect( + ctx.DocumentConversionManager.promises + .convertDocumentToLaTeXZipArchive + ).not.to.have.been.called + }) + }) + describe('unsuccessfully', async function () { beforeEach(async function (ctx) { - ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive = + ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive = sinon.stub().rejects(new Error('Conversion failed')) await new Promise(resolve => { @@ -532,14 +621,15 @@ describe('ProjectUploadController', function () { expect(data.success).to.be.false resolve() } - ctx.ProjectUploadController.importDocx(ctx.req, ctx.res) + ctx.ProjectUploadController.importDocument(ctx.req, ctx.res) }) }) it('should call the DocumentConversionManager to convert the file', function (ctx) { expect( - ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive - ).to.have.been.calledWith(ctx.req.file.path, ctx.user_id) + ctx.DocumentConversionManager.promises + .convertDocumentToLaTeXZipArchive + ).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'docx') }) it('should unlink the uploaded file', function (ctx) { @@ -553,7 +643,7 @@ describe('ProjectUploadController', function () { describe('when the converted archive is too large', async function () { beforeEach(async function (ctx) { - ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive = + ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive = sinon.stub().rejects(new FileTooLargeError('file too large')) await new Promise(resolve => { @@ -564,7 +654,7 @@ describe('ProjectUploadController', function () { }) resolve() } - ctx.ProjectUploadController.importDocx(ctx.req, ctx.res) + ctx.ProjectUploadController.importDocument(ctx.req, ctx.res) }) }) diff --git a/services/web/types/assets.d.ts b/services/web/types/assets.d.ts index 458256dab7..fca1c29b40 100644 --- a/services/web/types/assets.d.ts +++ b/services/web/types/assets.d.ts @@ -27,3 +27,5 @@ declare module '*.txt' { const src: string export default src } + +declare module '*.css' {}