Merge pull request #32857 from overleaf/ds-pandoc-import-md

[WEB + CLSI] Import markdown files using pandoc

GitOrigin-RevId: adad7831ddb13a8fcb8063871166bde13cbbf1b6

services/clsi/app.js

@@ -125,10 +125,20 @@ app.get(
 )
 
 // Conversion endpoints
+// Keep old route for backwards compatibility during CLSI/web deploy transition
 app.post(
   '/convert/docx-to-latex',
   FileUploadMiddleware.multerMiddleware,
-  ConversionController.convertDocxToLaTeX
+  (req, res, next) => {
+    req.query.type = 'docx'
+    next()
+  },
+  ConversionController.convertDocumentToLaTeX
+)
+app.post(
+  '/convert/document-to-latex',
+  FileUploadMiddleware.multerMiddleware,
+  ConversionController.convertDocumentToLaTeX
 )
 app.post(
   '/project/:project_id/user/:user_id/download/project-to-document',

services/clsi/app/js/ConversionController.js

@@ -11,19 +11,25 @@ import Path from 'node:path'
 
 const SUPPORTED_CONVERSION_TYPES = new Map([['docx', 'docx']])
 
-async function convertDocxToLaTeX(req, res) {
+async function convertDocumentToLaTeX(req, res) {
   const { path } = req.file
+  const conversionType = req.query.type
   if (!Settings.enablePandocConversions) {
     await fs.unlink(path).catch(() => {})
     return res.sendStatus(404)
   }
-  logger.debug({ path }, 'received file for conversion')
+  if (!conversionType || !['docx', 'markdown'].includes(conversionType)) {
+    await fs.unlink(path).catch(() => {})
+    return res.sendStatus(400)
+  }
+  logger.debug({ path, conversionType }, 'received file for conversion')
   const conversionId = crypto.randomUUID()
   let zipPath
   try {
-    zipPath = await ConversionManager.promises.convertDocxToLaTeXWithLock(
+    zipPath = await ConversionManager.promises.convertToLaTeXWithLock(
       conversionId,
-      path
+      path,
+      conversionType
     )
   } finally {
     await fs.unlink(path).catch(() => {})
@@ -98,6 +104,6 @@ async function convertProjectToDocument(req, res) {
 }
 
 export default {
-  convertDocxToLaTeX: expressify(convertDocxToLaTeX),
+  convertDocumentToLaTeX: expressify(convertDocumentToLaTeX),
   convertProjectToDocument: expressify(convertProjectToDocument),
 }

services/clsi/app/js/ConversionManager.js

@@ -6,19 +6,44 @@ import CommandRunner from './CommandRunner.js'
 import LockManager from './LockManager.js'
 import OError from '@overleaf/o-error'
 
-async function convertDocxToLaTeXWithLock(conversionId, inputPath) {
+const CONVERSION_CONFIGS = {
+  docx: {
+    inputFilename: 'input.docx',
+    pandocArgs: ['--extract-media=.', '--from', 'docx+citations', '--citeproc'],
+  },
+  markdown: {
+    inputFilename: 'input.md',
+    pandocArgs: ['--from', 'markdown'],
+  },
+}
+
+async function convertToLaTeXWithLock(conversionId, inputPath, conversionType) {
   const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
   const lock = LockManager.acquire(conversionDir)
   try {
-    return await convertDocxToLaTeX(conversionId, conversionDir, inputPath)
+    return await convertToLaTeX(
+      conversionId,
+      conversionDir,
+      inputPath,
+      conversionType
+    )
   } finally {
     lock.release()
   }
 }
 
-async function convertDocxToLaTeX(conversionId, conversionDir, inputPath) {
+async function convertToLaTeX(
+  conversionId,
+  conversionDir,
+  inputPath,
+  conversionType
+) {
+  const config = CONVERSION_CONFIGS[conversionType]
+  if (!config) {
+    throw new OError('unsupported conversion type', { conversionType })
+  }
   await fs.mkdir(conversionDir, { recursive: true })
-  const newSourcePath = Path.join(conversionDir, 'input.docx')
+  const newSourcePath = Path.join(conversionDir, config.inputFilename)
   await fs.copyFile(inputPath, newSourcePath)
   const outputName = crypto.randomUUID() + '.zip'
 
@@ -31,16 +56,13 @@ async function convertDocxToLaTeX(conversionId, conversionDir, inputPath) {
       conversionId,
       [
         'pandoc',
-        'input.docx',
+        config.inputFilename,
         '--output',
         'main.tex',
-        '--extract-media=.',
-        '--from',
-        'docx+citations',
         '--to',
         'latex',
-        '--citeproc',
         '--standalone',
+        ...config.pandocArgs,
       ],
       conversionDir,
       Settings.pandocImage,
@@ -164,7 +186,7 @@ async function convertLaTeXToDocumentInDir(
 
 export default {
   promises: {
-    convertDocxToLaTeXWithLock,
+    convertToLaTeXWithLock,
     convertLaTeXToDocumentInDirWithLock,
   },
 }

services/clsi/test/acceptance/js/ConversionTests.js

@@ -25,7 +25,7 @@ describe('Conversions', function () {
       const outputStream = fs.createWriteStream(
         '/tmp/clsi_acceptance_tests_' + crypto.randomUUID() + '.zip'
       )
-      const stream = await Client.convertDocx(sourcePath)
+      const stream = await Client.convertDocument(sourcePath, 'docx')
       await pipeline(stream, outputStream)
 
       await new Promise((resolve, reject) => {
@@ -77,7 +77,8 @@ describe('Conversions', function () {
         import.meta.dirname,
         '../fixtures/minimal.pdf'
       )
-      await expect(Client.convertDocx(sourcePath)).to.eventually.be.rejected
+      await expect(Client.convertDocument(sourcePath, 'docx')).to.eventually.be
+        .rejected
     })
   })
 })

services/clsi/test/acceptance/js/helpers/Client.js

@@ -30,10 +30,10 @@ function compile(projectId, data) {
   })
 }
 
-async function convertDocx(path) {
+async function convertDocument(path, type) {
   const formData = new FormData()
   formData.append('qqfile', fs.createReadStream(path))
-  return await fetchStream(`${host}/convert/docx-to-latex`, {
+  return await fetchStream(`${host}/convert/document-to-latex?type=${type}`, {
     method: 'POST',
     body: formData,
   })
@@ -202,7 +202,7 @@ function smokeTest() {
 export default {
   randomId,
   compile,
-  convertDocx,
+  convertDocument,
   stopCompile,
   clearCache,
   getOutputFile,

services/clsi/test/unit/js/ConversionController.test.js

@@ -22,7 +22,7 @@ describe('ConversionController', function () {
     ctx.parsedRequest = { rootResourcePath: 'main.tex' }
     ctx.ConversionManager = {
       promises: {
-        convertDocxToLaTeXWithLock: sinon.stub().resolves(ctx.zipPath),
+        convertToLaTeXWithLock: sinon.stub().resolves(ctx.zipPath),
         convertLaTeXToDocumentInDirWithLock: sinon
           .stub()
           .resolves(ctx.documentPath),
@@ -86,16 +86,17 @@ describe('ConversionController', function () {
     ctx.ConversionController = (await import(MODULE_PATH)).default
   })
 
-  describe('convertDocxToLaTeX', function () {
+  describe('convertDocumentToLaTeX', function () {
     describe('when conversions are disabled', function () {
       beforeEach(async function (ctx) {
         ctx.Settings.enablePandocConversions = false
         ctx.req = {
           file: { path: '/path/to/uploaded/file.docx' },
+          query: { type: 'docx' },
         }
         ctx.res.sendStatus = sinon.stub()
 
-        await ctx.ConversionController.convertDocxToLaTeX(ctx.req, ctx.res)
+        await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
       })
 
       it('should remove the uploaded file', function (ctx) {
@@ -108,7 +109,59 @@ describe('ConversionController', function () {
 
       it('should not call the conversion manager', function (ctx) {
         sinon.assert.notCalled(
-          ctx.ConversionManager.promises.convertDocxToLaTeXWithLock
+          ctx.ConversionManager.promises.convertToLaTeXWithLock
+        )
+      })
+    })
+
+    describe('when conversionType is missing', function () {
+      beforeEach(async function (ctx) {
+        ctx.req = {
+          file: { path: '/path/to/uploaded/file.docx' },
+          query: {},
+        }
+        ctx.res.sendStatus = sinon.stub()
+
+        await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
+      })
+
+      it('should remove the uploaded file', function (ctx) {
+        sinon.assert.calledWith(ctx.fs.unlink, ctx.req.file.path)
+      })
+
+      it('should return 400', function (ctx) {
+        sinon.assert.calledWith(ctx.res.sendStatus, 400)
+      })
+
+      it('should not call the conversion manager', function (ctx) {
+        sinon.assert.notCalled(
+          ctx.ConversionManager.promises.convertToLaTeXWithLock
+        )
+      })
+    })
+
+    describe('when conversionType is unsupported', function () {
+      beforeEach(async function (ctx) {
+        ctx.req = {
+          file: { path: '/path/to/uploaded/file.docx' },
+          query: { type: 'invalid' },
+        }
+        ctx.res.sendStatus = sinon.stub()
+
+        await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
+      })
+
+      it('should remove the uploaded file', function (ctx) {
+        sinon.assert.calledWith(ctx.fs.unlink, ctx.req.file.path)
+      })
+
+      it('should return 400', function (ctx) {
+        sinon.assert.calledWith(ctx.res.sendStatus, 400)
+      })
+
+      it('should not call the conversion manager', function (ctx) {
+        sinon.assert.notCalled(
+          ctx.ConversionManager.promises.convertToLaTeXWithLock
         )
       })
     })
@@ -117,18 +170,20 @@ describe('ConversionController', function () {
       beforeEach(async function (ctx) {
         ctx.req = {
           file: { path: '/path/to/uploaded/file.docx' },
+          query: { type: 'docx' },
         }
 
-        await ctx.ConversionController.convertDocxToLaTeX(ctx.req, ctx.res)
+        await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
       })
 
-      it('should call the conversion manager with the uploaded file path', function (ctx) {
+      it('should call the conversion manager with the uploaded file path and type', function (ctx) {
         sinon.assert.calledWith(
-          ctx.ConversionManager.promises.convertDocxToLaTeXWithLock,
+          ctx.ConversionManager.promises.convertToLaTeXWithLock,
           sinon.match(
             /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/
           ),
-          ctx.req.file.path
+          ctx.req.file.path,
+          'docx'
         )
       })
 
@@ -160,6 +215,28 @@ describe('ConversionController', function () {
       })
     })
 
+    describe('with conversionType=markdown', function () {
+      beforeEach(async function (ctx) {
+        ctx.req = {
+          file: { path: '/path/to/uploaded/file.md' },
+          query: { type: 'markdown' },
+        }
+
+        await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
+      })
+
+      it('should call the conversion manager with the uploaded file path and markdown type', function (ctx) {
+        sinon.assert.calledWith(
+          ctx.ConversionManager.promises.convertToLaTeXWithLock,
+          sinon.match(
+            /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/
+          ),
+          ctx.req.file.path,
+          'markdown'
+        )
+      })
+    })
+
     describe('unsuccessfully', function () {
       describe('on streaming error', function () {
         it('should propagate the error and still clean up', async function (ctx) {
@@ -169,10 +246,13 @@ describe('ConversionController', function () {
           res.attachment = sinon.stub()
           res.setHeader = sinon.stub()
 
-          const req = { file: { path: '/path/to/uploaded/file.docx' } }
+          const req = {
+            file: { path: '/path/to/uploaded/file.docx' },
+            query: { type: 'docx' },
+          }
 
           await expect(
-            ctx.ConversionController.convertDocxToLaTeX(req, res)
+            ctx.ConversionController.convertDocumentToLaTeX(req, res)
           ).to.be.rejectedWith('mock stream error')
 
           sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir)

services/clsi/test/unit/js/ConversionManager.test.js

@@ -36,7 +36,6 @@ describe('ConversionManager', function () {
     }
 
     ctx.conversionId = 'test-conversion-id'
-    ctx.inputPath = '/path/to/input.docx'
     ctx.conversionDir = '/compiles/test-conversion-id'
     ctx.outputPath = '/compiles/test-conversion-id/output-uuid.zip'
 
@@ -65,188 +64,287 @@ describe('ConversionManager', function () {
     ctx.uuidStub.restore()
   })
 
-  describe('convertDocxToLaTeXWithLock', function () {
+  describe('convertToLaTeXWithLock', function () {
-    describe('general behavior', function () {
+    describe('with conversionType=docx', function () {
-      beforeEach(async function (ctx) {
+      beforeEach(function (ctx) {
-        ctx.result =
+        ctx.inputPath = '/path/to/input.docx'
-          await ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
+      })
-            ctx.conversionId,
+
-            ctx.inputPath
+      describe('file setup and pandoc args', function () {
+        beforeEach(async function (ctx) {
+          ctx.result =
+            await ctx.ConversionManager.promises.convertToLaTeXWithLock(
+              ctx.conversionId,
+              ctx.inputPath,
+              'docx'
+            )
+        })
+
+        it('should acquire a lock', async function (ctx) {
+          sinon.assert.calledWith(ctx.LockManager.acquire, ctx.conversionDir)
+        })
+
+        it('should copy the input file to the conversion directory with docx filename', async function (ctx) {
+          sinon.assert.calledWith(ctx.fs.mkdir, ctx.conversionDir, {
+            recursive: true,
+          })
+          sinon.assert.calledWith(
+            ctx.fs.copyFile,
+            ctx.inputPath,
+            Path.join(ctx.conversionDir, 'input.docx')
           )
-      })
-
-      it('should acquire a lock', async function (ctx) {
-        sinon.assert.calledWith(ctx.LockManager.acquire, ctx.conversionDir)
-      })
-
-      it('should copy the input file to the conversion directory', async function (ctx) {
-        sinon.assert.calledWith(ctx.fs.mkdir, ctx.conversionDir, {
-          recursive: true,
-        })
-        sinon.assert.calledWith(
-          ctx.fs.copyFile,
-          ctx.inputPath,
-          Path.join(ctx.conversionDir, 'input.docx')
-        )
-      })
-
-      it('should convert conversion timeout to milliseconds', async function (ctx) {
-        expect(ctx.CommandRunner.promises.run.firstCall.args[4]).toBe(60_000)
-        expect(ctx.CommandRunner.promises.run.secondCall.args[4]).toBe(60_000)
-      })
-
-      it('should run pandoc followed by zip in the conversion directory', function (ctx) {
-        expect(ctx.CommandRunner.promises.run.callCount).toBe(2)
-        expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([
-          ctx.conversionId,
-          [
-            'pandoc',
-            'input.docx',
-            '--output',
-            'main.tex',
-            '--extract-media=.',
-            '--from',
-            'docx+citations',
-            '--to',
-            'latex',
-            '--citeproc',
-            '--standalone',
-          ],
-          ctx.conversionDir,
-          ctx.Settings.pandocImage,
-          60_000,
-          {},
-          'conversions',
-        ])
-        expect(ctx.CommandRunner.promises.run.secondCall.args).toEqual([
-          ctx.conversionId,
-          ['zip', '-r', 'output-uuid.zip', '.'],
-          ctx.conversionDir,
-          ctx.Settings.pandocImage,
-          60_000,
-          {},
-          'conversions',
-        ])
-      })
-    })
-
-    describe('successful conversion', function () {
-      beforeEach(async function (ctx) {
-        ctx.CommandRunner.promises.run.resolves({
-          stdout: 'mock-stdout',
-          stderr: 'mock-stderr',
-          exitCode: 0,
         })
 
-        ctx.result =
+        it('should convert conversion timeout to milliseconds', async function (ctx) {
-          await ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
+          expect(ctx.CommandRunner.promises.run.firstCall.args[4]).toBe(60_000)
+          expect(ctx.CommandRunner.promises.run.secondCall.args[4]).toBe(60_000)
+        })
+
+        it('should run pandoc with docx args followed by zip', function (ctx) {
+          expect(ctx.CommandRunner.promises.run.callCount).toBe(2)
+          expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([
             ctx.conversionId,
-            ctx.inputPath
+            [
-          )
+              'pandoc',
-      })
+              'input.docx',
-
+              '--output',
-      it('should remove the source document after conversion', async function (ctx) {
+              'main.tex',
-        sinon.assert.calledWith(
+              '--to',
-          ctx.fs.unlink,
+              'latex',
-          Path.join(ctx.conversionDir, 'input.docx')
+              '--standalone',
-        )
+              '--extract-media=.',
-      })
+              '--from',
-
+              'docx+citations',
-      it('should return the conversion directory', function (ctx) {
+              '--citeproc',
-        expect(ctx.result).toBe(ctx.outputPath)
+            ],
-      })
+            ctx.conversionDir,
-
+            ctx.Settings.pandocImage,
-      it('should release the lock', function (ctx) {
+            60_000,
-        sinon.assert.called(ctx.lock.release)
+            {},
-      })
+            'conversions',
-    })
+          ])
-
+          expect(ctx.CommandRunner.promises.run.secondCall.args).toEqual([
-    describe('unsuccessful conversion (exitcode)', function () {
-      beforeEach(async function (ctx) {
-        ctx.CommandRunner.promises.run.resolves({
-          stdout: 'mock-stdout',
-          stderr: 'mock-stderr',
-          exitCode: 63,
-        })
-
-        await expect(
-          ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
             ctx.conversionId,
-            ctx.inputPath
+            ['zip', '-r', 'output-uuid.zip', '.'],
-          )
+            ctx.conversionDir,
-        ).to.be.rejectedWith('pandoc conversion failed')
+            ctx.Settings.pandocImage,
-      })
+            60_000,
-
+            {},
-      it('should remove the entire conversion directory', async function (ctx) {
+            'conversions',
-        sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
+          ])
-          force: true,
-          recursive: true,
         })
       })
 
-      it('should release the lock', function (ctx) {
+      describe('successful conversion', function () {
-        sinon.assert.called(ctx.lock.release)
+        beforeEach(async function (ctx) {
-      })
+          ctx.CommandRunner.promises.run.resolves({
-    })
+            stdout: 'mock-stdout',
-
+            stderr: 'mock-stderr',
-    describe('unsuccessful compression (exitcode)', function () {
-      beforeEach(async function (ctx) {
-        ctx.CommandRunner.promises.run
-          .onFirstCall()
-          .resolves({
-            stdout: 'mock-pandoc-stdout',
-            stderr: 'mock-pandoc-stderr',
             exitCode: 0,
           })
-          .onSecondCall()
-          .resolves({
-            stdout: 'mock-zip-stdout',
-            stderr: 'mock-zip-stderr',
-            exitCode: 12,
-          })
 
-        await expect(
+          ctx.result =
-          ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
+            await ctx.ConversionManager.promises.convertToLaTeXWithLock(
-            ctx.conversionId,
+              ctx.conversionId,
-            ctx.inputPath
+              ctx.inputPath,
+              'docx'
+            )
+        })
+
+        it('should remove the source document after conversion', async function (ctx) {
+          sinon.assert.calledWith(
+            ctx.fs.unlink,
+            Path.join(ctx.conversionDir, 'input.docx')
           )
-        ).to.be.rejectedWith('pandoc conversion failed')
+        })
-      })
 
-      it('should remove the entire conversion directory', async function (ctx) {
+        it('should return the output zip path', function (ctx) {
-        sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
+          expect(ctx.result).toBe(ctx.outputPath)
-          force: true,
+        })
-          recursive: true,
+
+        it('should release the lock', function (ctx) {
+          sinon.assert.called(ctx.lock.release)
         })
       })
 
-      it('should release the lock', function (ctx) {
+      describe('unsuccessful conversion (exitcode)', function () {
-        sinon.assert.called(ctx.lock.release)
+        beforeEach(async function (ctx) {
+          ctx.CommandRunner.promises.run.resolves({
+            stdout: 'mock-stdout',
+            stderr: 'mock-stderr',
+            exitCode: 63,
+          })
+
+          await expect(
+            ctx.ConversionManager.promises.convertToLaTeXWithLock(
+              ctx.conversionId,
+              ctx.inputPath,
+              'docx'
+            )
+          ).to.be.rejectedWith('pandoc conversion failed')
+        })
+
+        it('should remove the entire conversion directory', async function (ctx) {
+          sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
+            force: true,
+            recursive: true,
+          })
+        })
+
+        it('should release the lock', function (ctx) {
+          sinon.assert.called(ctx.lock.release)
+        })
+      })
+
+      describe('unsuccessful compression (exitcode)', function () {
+        beforeEach(async function (ctx) {
+          ctx.CommandRunner.promises.run
+            .onFirstCall()
+            .resolves({
+              stdout: 'mock-pandoc-stdout',
+              stderr: 'mock-pandoc-stderr',
+              exitCode: 0,
+            })
+            .onSecondCall()
+            .resolves({
+              stdout: 'mock-zip-stdout',
+              stderr: 'mock-zip-stderr',
+              exitCode: 12,
+            })
+
+          await expect(
+            ctx.ConversionManager.promises.convertToLaTeXWithLock(
+              ctx.conversionId,
+              ctx.inputPath,
+              'docx'
+            )
+          ).to.be.rejectedWith('pandoc conversion failed')
+        })
+
+        it('should remove the entire conversion directory', async function (ctx) {
+          sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
+            force: true,
+            recursive: true,
+          })
+        })
+
+        it('should release the lock', function (ctx) {
+          sinon.assert.called(ctx.lock.release)
+        })
+      })
+
+      describe('unsuccessful conversion (throws)', function () {
+        beforeEach(async function (ctx) {
+          ctx.CommandRunner.promises.run.rejects(
+            new Error('mock conversion error')
+          )
+          await expect(
+            ctx.ConversionManager.promises.convertToLaTeXWithLock(
+              ctx.conversionId,
+              ctx.inputPath,
+              'docx'
+            )
+          ).to.be.rejectedWith('pandoc conversion failed')
+        })
+
+        it('should remove the entire conversion directory', async function (ctx) {
+          sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
+            force: true,
+            recursive: true,
+          })
+        })
+
+        it('should release the lock', function (ctx) {
+          sinon.assert.called(ctx.lock.release)
+        })
       })
     })
 
-    describe('unsuccessful conversion (throws)', function () {
+    describe('with conversionType=markdown', function () {
-      beforeEach(async function (ctx) {
+      beforeEach(function (ctx) {
-        ctx.CommandRunner.promises.run.rejects(
+        ctx.inputPath = '/path/to/input.md'
-          new Error('mock conversion error')
-        )
-        await expect(
-          ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
-            ctx.conversionId,
-            ctx.inputPath
-          )
-        ).to.be.rejectedWith('pandoc conversion failed')
       })
 
-      it('should remove the entire conversion directory', async function (ctx) {
+      describe('file setup and pandoc args', function () {
-        sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
+        beforeEach(async function (ctx) {
-          force: true,
+          ctx.result =
-          recursive: true,
+            await ctx.ConversionManager.promises.convertToLaTeXWithLock(
+              ctx.conversionId,
+              ctx.inputPath,
+              'markdown'
+            )
+        })
+
+        it('should copy the input file to the conversion directory with md filename', async function (ctx) {
+          sinon.assert.calledWith(ctx.fs.mkdir, ctx.conversionDir, {
+            recursive: true,
+          })
+          sinon.assert.calledWith(
+            ctx.fs.copyFile,
+            ctx.inputPath,
+            Path.join(ctx.conversionDir, 'input.md')
+          )
+        })
+
+        it('should run pandoc with markdown args followed by zip', function (ctx) {
+          expect(ctx.CommandRunner.promises.run.callCount).toBe(2)
+          expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([
+            ctx.conversionId,
+            [
+              'pandoc',
+              'input.md',
+              '--output',
+              'main.tex',
+              '--to',
+              'latex',
+              '--standalone',
+              '--from',
+              'markdown',
+            ],
+            ctx.conversionDir,
+            ctx.Settings.pandocImage,
+            60_000,
+            {},
+            'conversions',
+          ])
+          expect(ctx.CommandRunner.promises.run.secondCall.args).toEqual([
+            ctx.conversionId,
+            ['zip', '-r', 'output-uuid.zip', '.'],
+            ctx.conversionDir,
+            ctx.Settings.pandocImage,
+            60_000,
+            {},
+            'conversions',
+          ])
         })
       })
 
-      it('should release the lock', function (ctx) {
+      describe('successful conversion', function () {
-        sinon.assert.called(ctx.lock.release)
+        beforeEach(async function (ctx) {
+          ctx.CommandRunner.promises.run.resolves({
+            stdout: 'mock-stdout',
+            stderr: 'mock-stderr',
+            exitCode: 0,
+          })
+
+          ctx.result =
+            await ctx.ConversionManager.promises.convertToLaTeXWithLock(
+              ctx.conversionId,
+              ctx.inputPath,
+              'markdown'
+            )
+        })
+
+        it('should remove the source document after conversion', async function (ctx) {
+          sinon.assert.calledWith(
+            ctx.fs.unlink,
+            Path.join(ctx.conversionDir, 'input.md')
+          )
+        })
+
+        it('should return the output zip path', function (ctx) {
+          expect(ctx.result).toBe(ctx.outputPath)
+        })
       })
     })
   })

services/web/app/src/Features/Project/ProjectListController.mjs

@@ -531,6 +531,7 @@ async function projectListPage(req, res, next) {
     // Split tests that will be made available to the frontend
     'import-docx',
     'overleaf-library',
+    'import-markdown',
   ].filter(Boolean)
 
   await Promise.all(

services/web/app/src/Features/Uploads/DocumentConversionManager.mjs

@@ -11,20 +11,26 @@ import OError from '@overleaf/o-error'
 import FormData from 'form-data'
 import { FileTooLargeError } from '../Errors/Errors.js'
 
-async function convertDocxToLaTeXZipArchive(path, userId) {
+async function convertDocumentToLaTeXZipArchive(path, userId, conversionType) {
   const clsiUrl = new URL(Settings.apis.clsi.url)
   const limits = await CompileManager.promises._getUserCompileLimits(userId)
 
-  clsiUrl.pathname = '/convert/docx-to-latex'
+  // Uncomment this and remove the line below when the deploy is done.
+  // clsiUrl.pathname = '/convert/document-to-latex'
+  clsiUrl.pathname =
+    conversionType === 'docx'
+      ? '/convert/docx-to-latex'
+      : '/convert/document-to-latex'
   clsiUrl.searchParams.set('compileBackendClass', limits.compileBackendClass)
   clsiUrl.searchParams.set('compileGroup', limits.compileGroup)
+  clsiUrl.searchParams.set('type', conversionType)
 
   const formData = new FormData()
   formData.append('qqfile', fs.createReadStream(path))
 
   logger.debug(
-    { clsiUrl: clsiUrl.toString() },
+    { clsiUrl: clsiUrl.toString(), conversionType },
-    'sending docx to CLSI for conversion'
+    'sending document to CLSI for conversion'
   )
 
   const outputFileName = crypto.randomUUID() + '_document-conversion' + '.zip'
@@ -99,7 +105,7 @@ async function convertProjectToDocument(projectId, userId, type) {
 
 export default {
   promises: {
-    convertDocxToLaTeXZipArchive,
+    convertDocumentToLaTeXZipArchive,
     convertProjectToDocument,
   },
 }

services/web/app/src/Features/Uploads/ProjectUploadController.mjs

@@ -178,16 +178,21 @@ async function uploadFile(req, res, next) {
  * @param {any} res
  * @param {any} next
  */
-async function importDocx(req, res, next) {
+async function importDocument(req, res, next) {
   const userId = SessionManager.getLoggedInUserId(req.session)
-  logger.debug({ path: req.file?.path, userId }, 'importing docx file')
   const { path } = req.file
-  const name = Path.basename(req.body.name, '.docx')
+  const conversionType = req.query.type
+  if (!['docx', 'markdown'].includes(conversionType)) {
+    return res.status(400).json({ success: false, error: 'invalid_type' })
+  }
+  const name = Path.basename(req.body.name, Path.extname(req.body.name))
+  logger.debug({ path, userId, conversionType }, 'importing document file')
   try {
     const archivePath =
-      await DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
+      await DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
         path,
-        userId
+        userId,
+        conversionType
       )
     try {
       const project =
@@ -207,7 +212,7 @@ async function importDocx(req, res, next) {
       })
     }
   } catch (error) {
-    logger.error({ error }, 'error importing docx file')
+    logger.error({ error }, 'error importing document file')
     if (
       error instanceof FileTooLargeError ||
       error?.name === 'FileTooLargeError'
@@ -267,5 +272,5 @@ export default {
   uploadProject,
   uploadFile: expressify(uploadFile),
   multerMiddleware,
-  importDocx: expressify(importDocx),
+  importDocument: expressify(importDocument),
 }

services/web/app/src/Features/Uploads/UploadsRouter.mjs

@@ -28,12 +28,24 @@ export default {
     )
 
     if (Settings.enablePandocConversions) {
+      webRouter.post(
+        '/project/new/import-document',
+        AuthenticationController.requireLogin(),
+        RateLimiterMiddleware.rateLimit(rateLimiters.projectUpload),
+        ProjectUploadController.multerMiddleware,
+        ProjectUploadController.importDocument
+      )
+      // Keep old route for backwards compatibility with old frontends that haven't reloaded
       webRouter.post(
         '/project/new/import-docx',
         AuthenticationController.requireLogin(),
         RateLimiterMiddleware.rateLimit(rateLimiters.projectUpload),
         ProjectUploadController.multerMiddleware,
-        ProjectUploadController.importDocx
+        (req, res, next) => {
+          req.query.type = 'docx'
+          next()
+        },
+        ProjectUploadController.importDocument
       )
     }
 

services/web/frontend/extracted-translations.json

@@ -287,6 +287,7 @@
   "choose_a_custom_color": "",
   "choose_from_group_members": "",
   "choose_how_you_search_your_references": "",
+  "choose_markdown_file": "",
   "choose_which_experiments": "",
   "choose_word_document": "",
   "citation": "",
@@ -905,12 +906,13 @@
   "image_url": "",
   "image_width": "",
   "import_a_bibtex_file_from_your_provider_account": "",
+  "import_document_description": "",
   "import_existing_projects_from_github": "",
   "import_from_github": "",
   "import_idp_metadata": "",
+  "import_markdown_file": "",
   "import_to_sharelatex": "",
   "import_word_document": "",
-  "import_word_document_description": "",
   "imported_from_another_project_at_date": "",
   "imported_from_external_provider_at_date": "",
   "imported_from_mendeley_at_date": "",
@@ -1148,6 +1150,7 @@
   "manager": "",
   "managers_management": "",
   "managing_your_subscription": "",
+  "markdown_import_feedback_message": "",
   "marked_as_resolved": "",
   "math": "",
   "math_display": "",

services/web/frontend/js/features/ide-react/components/global-toasts.tsx

@@ -6,7 +6,7 @@ import { debugConsole } from '@/utils/debugging'
 import importOverleafModules from '../../../../macros/import-overleaf-module.macro'
 import { OLToastContainer } from '@/shared/components/ol/ol-toast-container'
 import clipboardToastGenerators from '@/features/source-editor/components/clipboard-toasts'
-import importDocxFeedbackToastGenerators from '@/features/project-list/components/new-project-button/import-docx-feedback-toast'
+import importDocumentFeedbackToastGenerators from '@/features/project-list/components/new-project-button/import-document-feedback-toast'
 import exportDocumentToastGenerators from '@/features/ide-react/components/toolbar/export-document-toasts'
 
 const moduleGeneratorsImport = importOverleafModules('toastGenerators') as {
@@ -29,7 +29,7 @@ type GlobalToastGenerator = (
 const GENERATOR_LIST: GlobalToastGeneratorEntry[] = [
   ...moduleGenerators.flat(),
   ...clipboardToastGenerators,
-  ...importDocxFeedbackToastGenerators,
+  ...importDocumentFeedbackToastGenerators,
   ...exportDocumentToastGenerators,
 ]
 const GENERATOR_MAP: Map<string, GlobalToastGenerator> = new Map(

services/web/frontend/js/features/ide-react/components/modals/modals.tsx

@@ -3,7 +3,7 @@ import ForceDisconnected from '@/features/ide-react/components/modals/force-disc
 import { UnsavedDocs } from '@/features/ide-react/components/unsaved-docs/unsaved-docs'
 import SystemMessages from '@/shared/components/system-messages'
 import ViewOnlyAccessModal from '@/features/share-project-modal/components/view-only-access-modal'
-import ProjectConvertedFromDocxModal from '@/features/ide-react/components/modals/project-converted-from-docx-modal'
+import ProjectConvertedFromDocumentModal from '@/features/ide-react/components/modals/project-converted-from-document-modal'
 
 export const Modals = memo(() => {
   return (
@@ -12,7 +12,7 @@ export const Modals = memo(() => {
       <UnsavedDocs />
       <SystemMessages />
       <ViewOnlyAccessModal />
-      <ProjectConvertedFromDocxModal />
+      <ProjectConvertedFromDocumentModal />
     </>
   )
 })

services/web/frontend/js/features/ide-react/components/modals/project-converted-from-document-modal.tsx

@@ -8,36 +8,35 @@ import {
 } from '@/shared/components/ol/ol-modal'
 import OLButton from '@/shared/components/ol/ol-button'
 import { useEffect, useState } from 'react'
-import { showImportDocxFeedbackToast } from '@/features/project-list/components/new-project-button/import-docx-feedback-toast'
+import { showImportDocumentFeedbackToast } from '@/features/project-list/components/new-project-button/import-document-feedback-toast'
 
-function ProjectConvertedFromDocxModal() {
+function ProjectConvertedFromDocumentModal() {
-  const [
+  const [convertedFrom, setConvertedFrom] = useState<string | null>(null)
-    showProjectConvertedFromDocxModal,
-    setShowProjectConvertedFromDocxModal,
-  ] = useState(false)
 
   useEffect(() => {
-    const query = window.location.search
+    const queryString = new URLSearchParams(window.location.search)
-    const queryString = new URLSearchParams(query)
+    const from = queryString.get('converted-from')
 
-    if (queryString.get('converted-from-docx') === 'true') {
+    if (from) {
-      setShowProjectConvertedFromDocxModal(true)
+      setConvertedFrom(from)
 
       // Clean the URL immediately so a refresh doesn't trigger the modal again,
       // but preserve other search params and the hash.
       const url = new URL(window.location.href)
-      url.searchParams.delete('converted-from-docx')
+      url.searchParams.delete('converted-from')
       window.history.replaceState(window.history.state, '', url.toString())
     }
   }, [])
 
   return (
     <>
-      {showProjectConvertedFromDocxModal && (
+      {convertedFrom && (
-        <ProjectConvertedFromDocxModalContent
+        <ProjectConvertedFromImportModalContent
           onHide={() => {
-            setShowProjectConvertedFromDocxModal(false)
+            setConvertedFrom(null)
-            showImportDocxFeedbackToast()
+            if (convertedFrom === 'docx' || convertedFrom === 'markdown') {
+              showImportDocumentFeedbackToast(convertedFrom)
+            }
           }}
         />
       )}
@@ -45,7 +44,7 @@ function ProjectConvertedFromDocxModal() {
   )
 }
 
-function ProjectConvertedFromDocxModalContent({
+function ProjectConvertedFromImportModalContent({
   onHide,
 }: {
   onHide: () => void
@@ -57,7 +56,7 @@ function ProjectConvertedFromDocxModalContent({
       show
       animation
       onHide={onHide}
-      id="converted-from-docx-modal"
+      id="converted-from-document-modal"
       backdrop="static"
     >
       <OLModalHeader>
@@ -73,4 +72,4 @@ function ProjectConvertedFromDocxModalContent({
   )
 }
 
-export default ProjectConvertedFromDocxModal
+export default ProjectConvertedFromDocumentModal

services/web/frontend/js/features/project-list/components/new-project-button.tsx

@@ -62,6 +62,9 @@ function NewProjectButton({
   const docxImportEnabled =
     useFeatureFlag('import-docx') &&
     getMeta('ol-ExposedSettings').enablePandocConversions
+  const markdownImportEnabled =
+    useFeatureFlag('import-markdown') &&
+    getMeta('ol-ExposedSettings').enablePandocConversions
   const sendTrackingEvent = useCallback(
     ({
       dropdownMenu,
@@ -228,6 +231,21 @@ function NewProjectButton({
               </DropdownItem>
             </li>
           )}
+          {markdownImportEnabled && (
+            <li role="none">
+              <DropdownItem
+                onClick={e =>
+                  handleModalMenuClick(e, {
+                    modalVariant: 'import_markdown',
+                    dropdownMenuEvent: 'import-markdown',
+                  })
+                }
+                trailingIcon={<MaterialIcon type="fiber_new" />}
+              >
+                {t('import_markdown_file')}
+              </DropdownItem>
+            </li>
+          )}
           <li role="none">
             {ImportProjectFromGithubMenu && (
               <ImportProjectFromGithubMenu

services/web/frontend/js/features/project-list/components/new-project-button/import-document-feedback-toast.tsx

@@ -0,0 +1,67 @@
+import { GlobalToastGeneratorEntry } from '@/features/ide-react/components/global-toasts'
+import { Trans } from 'react-i18next'
+
+const DocxImportFeedbackToast = () => (
+  <div>
+    <Trans
+      i18nKey="docx_import_feedback_message"
+      components={[
+        /* eslint-disable-next-line jsx-a11y/anchor-has-content, react/jsx-key */
+        <a
+          href="https://forms.gle/B1qrdiD983YcQCJA9"
+          target="_blank"
+          rel="noopener noreferrer"
+        />,
+      ]}
+    />
+  </div>
+)
+
+const MarkdownImportFeedbackToast = () => (
+  <div>
+    <Trans
+      i18nKey="markdown_import_feedback_message"
+      components={[
+        /* eslint-disable-next-line jsx-a11y/anchor-has-content, react/jsx-key */
+        <a
+          href="https://forms.gle/B1qrdiD983YcQCJA9"
+          target="_blank"
+          rel="noopener noreferrer"
+        />,
+      ]}
+    />
+  </div>
+)
+
+const generators: GlobalToastGeneratorEntry[] = [
+  {
+    key: 'import:docx-feedback',
+    generator: () => ({
+      content: <DocxImportFeedbackToast />,
+      type: 'info',
+      autoHide: false,
+      isDismissible: true,
+    }),
+  },
+  {
+    key: 'import:markdown-feedback',
+    generator: () => ({
+      content: <MarkdownImportFeedbackToast />,
+      type: 'info',
+      autoHide: false,
+      isDismissible: true,
+    }),
+  },
+]
+
+export default generators
+
+export const showImportDocumentFeedbackToast = (type: 'docx' | 'markdown') => {
+  const key =
+    type === 'markdown' ? 'import:markdown-feedback' : 'import:docx-feedback'
+  window.dispatchEvent(
+    new CustomEvent('ide:show-toast', {
+      detail: { key },
+    })
+  )
+}

services/web/frontend/js/features/project-list/components/new-project-button/import-document-modal.tsx

@@ -1,3 +1,4 @@
+import { useMemo } from 'react'
 import { Dashboard } from '@uppy/react'
 import { useTranslation } from 'react-i18next'
 import { useProjectUploader } from '../../hooks/use-project-uploader'
@@ -13,19 +14,39 @@ import '@uppy/core/dist/style.css'
 import '@uppy/dashboard/dist/style.css'
 import BetaBadgeIcon from '@/shared/components/beta-badge-icon'
 
-function ImportDocxModal({
+function ImportDocumentModal({
+  type,
   onHide,
   openProject,
 }: {
+  type: 'docx' | 'markdown'
   onHide: () => void
-  openProject: (id: string, isConvertedFromDocx?: boolean) => void
+  openProject: (id: string, convertedFrom?: string) => void
 }) {
   const { t } = useTranslation()
+  const IMPORT_CONFIGS = useMemo(
+    () => ({
+      docx: {
+        allowedFileTypes: ['.docx'],
+        title: t('choose_word_document'),
+        browseLabel: 'Select .docx file',
+        dragLabel: '%{browseFiles} or \n\n Drag .docx file',
+      },
+      markdown: {
+        allowedFileTypes: ['.md'],
+        title: t('choose_markdown_file'),
+        browseLabel: 'Select .md file',
+        dragLabel: '%{browseFiles} or \n\n Drag .md file',
+      },
+    }),
+    [t]
+  )
+  const config = IMPORT_CONFIGS[type]
 
   const uppy = useProjectUploader({
-    endpoint: '/project/new/import-docx',
+    endpoint: `/project/new/import-document?type=${type}`,
-    allowedFileTypes: ['.docx'],
+    allowedFileTypes: config.allowedFileTypes,
-    onSuccess: (projectId: string) => openProject(projectId, true),
+    onSuccess: (projectId: string) => openProject(projectId, type),
   })
 
   return (
@@ -36,16 +57,17 @@ function ImportDocxModal({
       id="upload-project-modal"
       backdrop="static"
     >
+      {/* TODO: make necessary changes here for import document modal */}
       <OLModalHeader>
-        <OLModalTitle as="h3" className="import-docx-modal-title">
+        <OLModalTitle as="h3" className="import-document-modal-title">
-          {t('choose_word_document')}
+          {config.title}
           <span className="beta-icon-wrapper">
             <BetaBadgeIcon />
           </span>
         </OLModalTitle>
       </OLModalHeader>
       <OLModalBody>
-        <p>{t('import_word_document_description')}</p>
+        <p>{t('import_document_description')}</p>
         <Dashboard
           uppy={uppy}
           proudlyDisplayPoweredByUppy={false}
@@ -55,8 +77,8 @@ function ImportDocxModal({
           height={300}
           locale={{
             strings: {
-              browseFiles: 'Select .docx file',
+              browseFiles: config.browseLabel,
-              dropPasteFiles: '%{browseFiles} or \n\n Drag .docx file',
+              dropPasteFiles: config.dragLabel,
             },
           }}
           className="project-list-upload-project-modal-uppy-dashboard"
@@ -71,4 +93,4 @@ function ImportDocxModal({
   )
 }
 
-export default ImportDocxModal
+export default ImportDocumentModal

services/web/frontend/js/features/project-list/components/new-project-button/import-docx-feedback-toast.tsx

@@ -1,42 +0,0 @@
-import { GlobalToastGeneratorEntry } from '@/features/ide-react/components/global-toasts'
-import { Trans } from 'react-i18next'
-
-const ImportDocxFeedbackToast = () => {
-  return (
-    <div>
-      <Trans
-        i18nKey="docx_import_feedback_message"
-        components={[
-          /* eslint-disable-next-line jsx-a11y/anchor-has-content, react/jsx-key */
-          <a
-            href="https://forms.gle/B1qrdiD983YcQCJA9"
-            target="_blank"
-            rel="noopener noreferrer"
-          />,
-        ]}
-      />
-    </div>
-  )
-}
-
-const generators: GlobalToastGeneratorEntry[] = [
-  {
-    key: 'import:docx-feedback',
-    generator: () => ({
-      content: <ImportDocxFeedbackToast />,
-      type: 'info',
-      autoHide: false,
-      isDismissible: true,
-    }),
-  },
-]
-
-export default generators
-
-export const showImportDocxFeedbackToast = () => {
-  window.dispatchEvent(
-    new CustomEvent('ide:show-toast', {
-      detail: { key: 'import:docx-feedback' },
-    })
-  )
-}

services/web/frontend/js/features/project-list/components/new-project-button/new-project-button-modal.tsx

@@ -7,7 +7,7 @@ import { FullSizeLoadingSpinner } from '@/shared/components/loading-spinner'
 import { useLocation } from '@/shared/hooks/use-location'
 
 const UploadProjectModal = lazy(() => import('./upload-project-modal'))
-const ImportDocxModal = lazy(() => import('./import-docx-modal'))
+const ImportDocumentModal = lazy(() => import('./import-document-modal'))
 
 export type NewProjectButtonModalVariant =
   | 'blank_project'
@@ -15,6 +15,7 @@ export type NewProjectButtonModalVariant =
   | 'upload_project'
   | 'import_from_github'
   | 'import_docx'
+  | 'import_markdown'
 
 type NewProjectButtonModalProps = {
   modal: Nullable<NewProjectButtonModalVariant>
@@ -32,9 +33,9 @@ function NewProjectButtonModal({ modal, onHide }: NewProjectButtonModalProps) {
   const location = useLocation()
 
   const openProject = useCallback(
-    (projectId: string, isConvertedFromDocx: boolean = false) => {
+    (projectId: string, convertedFrom?: string) => {
-      const url = isConvertedFromDocx
+      const url = convertedFrom
-        ? `/project/${projectId}?converted-from-docx=true`
+        ? `/project/${projectId}?converted-from=${convertedFrom}`
         : `/project/${projectId}`
 
       location.assign(url)
@@ -56,7 +57,21 @@ function NewProjectButtonModal({ modal, onHide }: NewProjectButtonModalProps) {
     case 'import_docx':
       return (
         <Suspense fallback={<FullSizeLoadingSpinner delay={500} />}>
-          <ImportDocxModal onHide={onHide} openProject={openProject} />
+          <ImportDocumentModal
+            type="docx"
+            onHide={onHide}
+            openProject={openProject}
+          />
+        </Suspense>
+      )
+    case 'import_markdown':
+      return (
+        <Suspense fallback={<FullSizeLoadingSpinner delay={500} />}>
+          <ImportDocumentModal
+            type="markdown"
+            onHide={onHide}
+            openProject={openProject}
+          />
         </Suspense>
       )
     case 'import_from_github':

services/web/frontend/js/features/project-list/components/welcome-message-new/welcome-message-create-new-project-dropdown.tsx

@@ -64,6 +64,9 @@ function WelcomeMessageCreateNewProjectDropdown({
   const docxImportEnabled =
     useFeatureFlag('import-docx') &&
     getMeta('ol-ExposedSettings').enablePandocConversions
+  const markdownImportEnabled =
+    useFeatureFlag('import-markdown') &&
+    getMeta('ol-ExposedSettings').enablePandocConversions
 
   const { isOverleaf } = getMeta('ol-ExposedSettings')
 
@@ -153,6 +156,20 @@ function WelcomeMessageCreateNewProjectDropdown({
             </DropdownItem>
           </li>
         )}
+        {markdownImportEnabled && (
+          <li role="none">
+            <DropdownItem
+              as="button"
+              onClick={e =>
+                handleDropdownItemClick(e, 'import_markdown', 'import-markdown')
+              }
+              tabIndex={-1}
+              trailingIcon={<MaterialIcon type="fiber_new" />}
+            >
+              {t('import_markdown_file')}
+            </DropdownItem>
+          </li>
+        )}
         {isOverleaf && (
           <li role="none">
             <DropdownItem

services/web/frontend/stylesheets/pages/project-list.scss

@@ -585,7 +585,7 @@ ul.project-list-filters {
   }
 }
 
-.import-docx-modal-title {
+.import-document-modal-title {
   display: flex;
   align-items: center;
 

services/web/locales/en.json

@@ -379,6 +379,7 @@
   "choose_a_new_password": "Choose a new password",
   "choose_from_group_members": "Choose from group members",
   "choose_how_you_search_your_references": "Choose how you search your references",
+  "choose_markdown_file": "Choose Markdown file",
   "choose_which_experiments": "Choose which experiments you’d like to try.",
   "choose_word_document": "Choose Word document",
   "choose_your_plan": "Choose your plan",
@@ -1174,12 +1175,13 @@
   "image_url": "Image URL",
   "image_width": "Image width",
   "import_a_bibtex_file_from_your_provider_account": "Import a BibTeX file from your __provider__ account",
+  "import_document_description": "Content will be imported from the selected document. Formatting may not be reproduced exactly.",
   "import_existing_projects_from_github": "Import existing projects from GitHub",
   "import_from_github": "Import from GitHub",
   "import_idp_metadata": "Import IdP metadata",
+  "import_markdown_file": "Import Markdown file",
   "import_to_sharelatex": "Import to __appName__",
   "import_word_document": "Import Word document",
-  "import_word_document_description": "Content will be imported from the selected document. Formatting may not be reproduced exactly.",
   "important_message": "Important message",
   "imported_from_another_project_at_date": "Imported from <0>Another project</0>/__sourceEntityPathHTML__, at __formattedDate__ __relativeDate__",
   "imported_from_external_provider_at_date": "Imported from <0>__shortenedUrlHTML__</0> at __formattedDate__ __relativeDate__",
@@ -1517,6 +1519,7 @@
   "managers_management": "Managers management",
   "managing_your_subscription": "Managing your subscription",
   "march": "March",
+  "markdown_import_feedback_message": "Importing Markdown files is a new feature. <0>Let us know what you think</0>",
   "marked_as_resolved": "Marked as resolved",
   "math": "Math",
   "math_display": "Math Display",

services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs

@@ -88,12 +88,74 @@ describe('DocumentConversionManager', function () {
     ctx.DocumentConversionManager = (await import(MODULE_PATH)).default
   })
 
-  describe('convertDocxToLaTeXZipArchive', function () {
+  describe('convertDocumentToLaTeXZipArchive', function () {
-    describe('successfully', function () {
+    describe('with conversionType=docx', function () {
+      describe('successfully', function () {
+        beforeEach(async function (ctx) {
+          ctx.path = '/path/to/input.docx'
+          ctx.userId = 'test-user-id'
+          ctx.response = {
+            headers: {
+              get: sinon.stub().returns(null),
+            },
+          }
+          ctx.response.headers.get.withArgs('Content-Length').returns('50')
+
+          ctx.fetchUtils.fetchStreamWithResponse.resolves({
+            stream: 'mocked-fetch-stream',
+            response: ctx.response,
+          })
+
+          ctx.result =
+            await ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
+              ctx.path,
+              ctx.userId,
+              'docx'
+            )
+        })
+
+        it('should call fetchStreamWithResponse with the correct URL and form data', function (ctx) {
+          const expectedUrl = new URL(ctx.Settings.apis.clsi.url)
+          // TODO: revert this to '/convert/document-to-latex' once the deploy is done (PR #32857)
+          expectedUrl.pathname = '/convert/docx-to-latex'
+          expectedUrl.searchParams.set(
+            'compileBackendClass',
+            'test-backend-class'
+          )
+          expectedUrl.searchParams.set('compileGroup', 'test-compile-group')
+          expectedUrl.searchParams.set('type', 'docx')
+
+          sinon.assert.calledWith(
+            ctx.fetchUtils.fetchStreamWithResponse,
+            sinon.match(url => url.toString() === expectedUrl.toString()),
+            {
+              method: 'POST',
+              body: sinon.match.instanceOf(FormData),
+              signal: sinon.match.instanceOf(AbortSignal),
+            }
+          )
+        })
+
+        it('should pipe result into the output file', function (ctx) {
+          sinon.assert.calledWith(
+            ctx.nodeStream.pipeline,
+            'mocked-fetch-stream',
+            'mocked-write-stream'
+          )
+        })
+
+        it('should return a path to the output file', function (ctx) {
+          expect(ctx.result).to.match(
+            /\/path\/to\/dump\/folder\/[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}_document-conversion\.zip/
+          )
+        })
+      })
+    })
+
+    describe('with conversionType=markdown', function () {
       beforeEach(async function (ctx) {
-        ctx.path = '/path/to/input.docx'
+        ctx.path = '/path/to/input.md'
         ctx.userId = 'test-user-id'
-        ctx.outputPath = '/path/to/output.zip'
         ctx.response = {
           headers: {
             get: sinon.stub().returns(null),
@@ -107,20 +169,22 @@ describe('DocumentConversionManager', function () {
         })
 
         ctx.result =
-          await ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
+          await ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
             ctx.path,
-            ctx.userId
+            ctx.userId,
+            'markdown'
           )
       })
 
-      it('should call fetchStreamWithResponse with the correct URL and form data', function (ctx) {
+      it('should call fetchStreamWithResponse with the correct URL including markdown type', function (ctx) {
         const expectedUrl = new URL(ctx.Settings.apis.clsi.url)
-        expectedUrl.pathname = '/convert/docx-to-latex'
+        expectedUrl.pathname = '/convert/document-to-latex'
         expectedUrl.searchParams.set(
           'compileBackendClass',
           'test-backend-class'
         )
         expectedUrl.searchParams.set('compileGroup', 'test-compile-group')
+        expectedUrl.searchParams.set('type', 'markdown')
 
         sinon.assert.calledWith(
           ctx.fetchUtils.fetchStreamWithResponse,
@@ -158,9 +222,10 @@ describe('DocumentConversionManager', function () {
         )
 
         await expect(
-          ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
+          ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
             ctx.path,
-            ctx.userId
+            ctx.userId,
+            'docx'
           )
         ).to.be.rejectedWith('document conversion failed')
       })
@@ -195,9 +260,10 @@ describe('DocumentConversionManager', function () {
         })
 
         await expect(
-          ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
+          ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
             ctx.path,
-            ctx.userId
+            ctx.userId,
+            'docx'
           )
         ).to.be.rejectedWith(sinon.match.instanceOf(FileTooLargeError))
       })

services/web/test/unit/src/Uploads/ProjectUploadController.test.mjs

@@ -48,7 +48,7 @@ describe('ProjectUploadController', function () {
     }
     ctx.DocumentConversionManager = {
       promises: {
-        convertDocxToLaTeXZipArchive: sinon.stub(),
+        convertDocumentToLaTeXZipArchive: sinon.stub(),
       },
     }
 
@@ -463,7 +463,7 @@ describe('ProjectUploadController', function () {
     })
   })
 
-  describe('importDocx', function () {
+  describe('importDocument', function () {
     beforeEach(async function (ctx) {
       ctx.req.file = {
         path: '/path/to/uploaded/file.docx',
@@ -471,13 +471,73 @@ describe('ProjectUploadController', function () {
       ctx.req.body = {
         name: 'file.docx',
       }
+      ctx.req.query = { type: 'docx' }
       ctx.archivePath = '/path/to/archive.zip'
       ctx.fsPromises.unlink = sinon.stub().resolves()
     })
 
-    describe('successfully', async function () {
+    describe('with conversionType=docx', async function () {
+      describe('successfully', async function () {
+        beforeEach(async function (ctx) {
+          ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
+            sinon.stub().resolves(ctx.archivePath)
+          ctx.ProjectUploadManager.promises.createProjectFromZipArchive = sinon
+            .stub()
+            .resolves({
+              _id: 'new-project-id',
+            })
+
+          await new Promise(resolve => {
+            ctx.res.json = data => {
+              expect(data.success).to.be.true
+              expect(data.project_id).to.equal('new-project-id')
+              resolve()
+            }
+            ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
+          })
+        })
+
+        it('should call the DocumentConversionManager with file path and type', function (ctx) {
+          expect(
+            ctx.DocumentConversionManager.promises
+              .convertDocumentToLaTeXZipArchive
+          ).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'docx')
+        })
+
+        it('should use the resulting archive to create a new project', function (ctx) {
+          expect(
+            ctx.ProjectUploadManager.promises.createProjectFromZipArchive
+          ).to.have.been.calledWith(ctx.user_id, 'file', ctx.archivePath)
+        })
+
+        it('should set the compiler to lualatex', function (ctx) {
+          expect(
+            ctx.ProjectOptionsHandler.promises.setCompiler
+          ).to.have.been.calledWith('new-project-id', 'lualatex')
+        })
+
+        it('should unlink the archive after creating the project', function (ctx) {
+          expect(ctx.fsPromises.unlink).to.have.been.calledWith(ctx.archivePath)
+        })
+
+        it('should unlink the uploaded file', function (ctx) {
+          expect(ctx.fsPromises.unlink).to.have.been.calledWith(
+            ctx.req.file.path
+          )
+        })
+      })
+    })
+
+    describe('with conversionType=markdown', async function () {
       beforeEach(async function (ctx) {
-        ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive =
+        ctx.req.file = {
+          path: '/path/to/uploaded/file.md',
+        }
+        ctx.req.body = {
+          name: 'file.md',
+        }
+        ctx.req.query = { type: 'markdown' }
+        ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
           sinon.stub().resolves(ctx.archivePath)
         ctx.ProjectUploadManager.promises.createProjectFromZipArchive = sinon
           .stub()
@@ -491,14 +551,15 @@ describe('ProjectUploadController', function () {
             expect(data.project_id).to.equal('new-project-id')
             resolve()
           }
-          ctx.ProjectUploadController.importDocx(ctx.req, ctx.res)
+          ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
         })
       })
 
-      it('should call the DocumentConversionManager to convert the file', function (ctx) {
+      it('should call the DocumentConversionManager with file path and markdown type', function (ctx) {
         expect(
-          ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive
+          ctx.DocumentConversionManager.promises
-        ).to.have.been.calledWith(ctx.req.file.path, ctx.user_id)
+            .convertDocumentToLaTeXZipArchive
+        ).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'markdown')
       })
 
       it('should use the resulting archive to create a new project', function (ctx) {
@@ -522,9 +583,37 @@ describe('ProjectUploadController', function () {
       })
     })
 
+    describe('with an invalid conversionType', async function () {
+      beforeEach(async function (ctx) {
+        ctx.req.query = { type: 'invalid' }
+
+        await new Promise(resolve => {
+          ctx.res.json = data => {
+            expect(data).to.deep.equal({
+              success: false,
+              error: 'invalid_type',
+            })
+            resolve()
+          }
+          ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
+        })
+      })
+
+      it('should return http 400', function (ctx) {
+        expect(ctx.res.statusCode).to.equal(400)
+      })
+
+      it('should not call DocumentConversionManager', function (ctx) {
+        expect(
+          ctx.DocumentConversionManager.promises
+            .convertDocumentToLaTeXZipArchive
+        ).not.to.have.been.called
+      })
+    })
+
     describe('unsuccessfully', async function () {
       beforeEach(async function (ctx) {
-        ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive =
+        ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
           sinon.stub().rejects(new Error('Conversion failed'))
 
         await new Promise(resolve => {
@@ -532,14 +621,15 @@ describe('ProjectUploadController', function () {
             expect(data.success).to.be.false
             resolve()
           }
-          ctx.ProjectUploadController.importDocx(ctx.req, ctx.res)
+          ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
         })
       })
 
       it('should call the DocumentConversionManager to convert the file', function (ctx) {
         expect(
-          ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive
+          ctx.DocumentConversionManager.promises
-        ).to.have.been.calledWith(ctx.req.file.path, ctx.user_id)
+            .convertDocumentToLaTeXZipArchive
+        ).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'docx')
       })
 
       it('should unlink the uploaded file', function (ctx) {
@@ -553,7 +643,7 @@ describe('ProjectUploadController', function () {
 
     describe('when the converted archive is too large', async function () {
       beforeEach(async function (ctx) {
-        ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive =
+        ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
           sinon.stub().rejects(new FileTooLargeError('file too large'))
 
         await new Promise(resolve => {
@@ -564,7 +654,7 @@ describe('ProjectUploadController', function () {
             })
             resolve()
           }
-          ctx.ProjectUploadController.importDocx(ctx.req, ctx.res)
+          ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
         })
       })
 

services/web/types/assets.d.ts

@@ -27,3 +27,5 @@ declare module '*.txt' {
   const src: string
   export default src
 }
+
+declare module '*.css' {}