agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-05-23 09:09:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #32857 from overleaf/ds-pandoc-import-md

Browse Source 
[WEB + CLSI] Import markdown files using pandoc

GitOrigin-RevId: adad7831ddb13a8fcb8063871166bde13cbbf1b6
This commit is contained in:
Mathias Jakobsen
2026-05-07 12:09:10 +01:00
committed by Copybot
parent 44efc9d745
commit eddcc5a42e
26 changed files with 813 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
services/clsi/app.js
View File

@@ -125,10 +125,20 @@ app.get(
)
// Conversion endpoints
// Keep old route for backwards compatibility during CLSI/web deploy transition
app.post(
'/convert/docx-to-latex',
FileUploadMiddleware.multerMiddleware,
ConversionController.convertDocxToLaTeX
(req, res, next) => {
req.query.type = 'docx'
next()
},
ConversionController.convertDocumentToLaTeX
)
app.post(
'/convert/document-to-latex',
FileUploadMiddleware.multerMiddleware,
ConversionController.convertDocumentToLaTeX
)
app.post(
'/project/:project_id/user/:user_id/download/project-to-document',

16
services/clsi/app/js/ConversionController.js
View File

@@ -11,19 +11,25 @@ import Path from 'node:path'
const SUPPORTED_CONVERSION_TYPES = new Map([['docx', 'docx']])
async function convertDocxToLaTeX(req, res) {
async function convertDocumentToLaTeX(req, res) {
const { path } = req.file
const conversionType = req.query.type
if (!Settings.enablePandocConversions) {
await fs.unlink(path).catch(() => {})
return res.sendStatus(404)
}
logger.debug({ path }, 'received file for conversion')
if (!conversionType || !['docx', 'markdown'].includes(conversionType)) {
await fs.unlink(path).catch(() => {})
return res.sendStatus(400)
}
logger.debug({ path, conversionType }, 'received file for conversion')
const conversionId = crypto.randomUUID()
let zipPath
try {
zipPath = await ConversionManager.promises.convertDocxToLaTeXWithLock(
zipPath = await ConversionManager.promises.convertToLaTeXWithLock(
conversionId,
path
path,
conversionType
)
} finally {
await fs.unlink(path).catch(() => {})
@@ -98,6 +104,6 @@ async function convertProjectToDocument(req, res) {
}
export default {
convertDocxToLaTeX: expressify(convertDocxToLaTeX),
convertDocumentToLaTeX: expressify(convertDocumentToLaTeX),
convertProjectToDocument: expressify(convertProjectToDocument),
}

42
services/clsi/app/js/ConversionManager.js
View File

@@ -6,19 +6,44 @@ import CommandRunner from './CommandRunner.js'
import LockManager from './LockManager.js'
import OError from '@overleaf/o-error'
async function convertDocxToLaTeXWithLock(conversionId, inputPath) {
const CONVERSION_CONFIGS = {
docx: {
inputFilename: 'input.docx',
pandocArgs: ['--extract-media=.', '--from', 'docx+citations', '--citeproc'],
},
markdown: {
inputFilename: 'input.md',
pandocArgs: ['--from', 'markdown'],
},
}
async function convertToLaTeXWithLock(conversionId, inputPath, conversionType) {
const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
const lock = LockManager.acquire(conversionDir)
try {
return await convertDocxToLaTeX(conversionId, conversionDir, inputPath)
return await convertToLaTeX(
conversionId,
conversionDir,
inputPath,
conversionType
)
} finally {
lock.release()
}
}
async function convertDocxToLaTeX(conversionId, conversionDir, inputPath) {
async function convertToLaTeX(
conversionId,
conversionDir,
inputPath,
conversionType
) {
const config = CONVERSION_CONFIGS[conversionType]
if (!config) {
throw new OError('unsupported conversion type', { conversionType })
}
await fs.mkdir(conversionDir, { recursive: true })
const newSourcePath = Path.join(conversionDir, 'input.docx')
const newSourcePath = Path.join(conversionDir, config.inputFilename)
await fs.copyFile(inputPath, newSourcePath)
const outputName = crypto.randomUUID() + '.zip'
@@ -31,16 +56,13 @@ async function convertDocxToLaTeX(conversionId, conversionDir, inputPath) {
conversionId,
[
'pandoc',
'input.docx',
config.inputFilename,
'--output',
'main.tex',
'--extract-media=.',
'--from',
'docx+citations',
'--to',
'latex',
'--citeproc',
'--standalone',
...config.pandocArgs,
],
conversionDir,
Settings.pandocImage,
@@ -164,7 +186,7 @@ async function convertLaTeXToDocumentInDir(
export default {
promises: {
convertDocxToLaTeXWithLock,
convertToLaTeXWithLock,
convertLaTeXToDocumentInDirWithLock,
},
}

5
services/clsi/test/acceptance/js/ConversionTests.js
View File

@@ -25,7 +25,7 @@ describe('Conversions', function () {
const outputStream = fs.createWriteStream(
'/tmp/clsi_acceptance_tests_' + crypto.randomUUID() + '.zip'
)
const stream = await Client.convertDocx(sourcePath)
const stream = await Client.convertDocument(sourcePath, 'docx')
await pipeline(stream, outputStream)
await new Promise((resolve, reject) => {
@@ -77,7 +77,8 @@ describe('Conversions', function () {
import.meta.dirname,
'../fixtures/minimal.pdf'
)
await expect(Client.convertDocx(sourcePath)).to.eventually.be.rejected
await expect(Client.convertDocument(sourcePath, 'docx')).to.eventually.be
.rejected
})
})
})

6
services/clsi/test/acceptance/js/helpers/Client.js
View File

@@ -30,10 +30,10 @@ function compile(projectId, data) {
})
}
async function convertDocx(path) {
async function convertDocument(path, type) {
const formData = new FormData()
formData.append('qqfile', fs.createReadStream(path))
return await fetchStream(`${host}/convert/docx-to-latex`, {
return await fetchStream(`${host}/convert/document-to-latex?type=${type}`, {
method: 'POST',
body: formData,
})
@@ -202,7 +202,7 @@ function smokeTest() {
export default {
randomId,
compile,
convertDocx,
convertDocument,
stopCompile,
clearCache,
getOutputFile,

100
services/clsi/test/unit/js/ConversionController.test.js
View File

@@ -22,7 +22,7 @@ describe('ConversionController', function () {
ctx.parsedRequest = { rootResourcePath: 'main.tex' }
ctx.ConversionManager = {
promises: {
convertDocxToLaTeXWithLock: sinon.stub().resolves(ctx.zipPath),
convertToLaTeXWithLock: sinon.stub().resolves(ctx.zipPath),
convertLaTeXToDocumentInDirWithLock: sinon
.stub()
.resolves(ctx.documentPath),
@@ -86,16 +86,17 @@ describe('ConversionController', function () {
ctx.ConversionController = (await import(MODULE_PATH)).default
})
describe('convertDocxToLaTeX', function () {
describe('convertDocumentToLaTeX', function () {
describe('when conversions are disabled', function () {
beforeEach(async function (ctx) {
ctx.Settings.enablePandocConversions = false
ctx.req = {
file: { path: '/path/to/uploaded/file.docx' },
query: { type: 'docx' },
}
ctx.res.sendStatus = sinon.stub()
await ctx.ConversionController.convertDocxToLaTeX(ctx.req, ctx.res)
await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
})
it('should remove the uploaded file', function (ctx) {
@@ -108,7 +109,59 @@ describe('ConversionController', function () {
it('should not call the conversion manager', function (ctx) {
sinon.assert.notCalled(
ctx.ConversionManager.promises.convertDocxToLaTeXWithLock
ctx.ConversionManager.promises.convertToLaTeXWithLock
)
})
})
describe('when conversionType is missing', function () {
beforeEach(async function (ctx) {
ctx.req = {
file: { path: '/path/to/uploaded/file.docx' },
query: {},
}
ctx.res.sendStatus = sinon.stub()
await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
})
it('should remove the uploaded file', function (ctx) {
sinon.assert.calledWith(ctx.fs.unlink, ctx.req.file.path)
})
it('should return 400', function (ctx) {
sinon.assert.calledWith(ctx.res.sendStatus, 400)
})
it('should not call the conversion manager', function (ctx) {
sinon.assert.notCalled(
ctx.ConversionManager.promises.convertToLaTeXWithLock
)
})
})
describe('when conversionType is unsupported', function () {
beforeEach(async function (ctx) {
ctx.req = {
file: { path: '/path/to/uploaded/file.docx' },
query: { type: 'invalid' },
}
ctx.res.sendStatus = sinon.stub()
await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
})
it('should remove the uploaded file', function (ctx) {
sinon.assert.calledWith(ctx.fs.unlink, ctx.req.file.path)
})
it('should return 400', function (ctx) {
sinon.assert.calledWith(ctx.res.sendStatus, 400)
})
it('should not call the conversion manager', function (ctx) {
sinon.assert.notCalled(
ctx.ConversionManager.promises.convertToLaTeXWithLock
)
})
})
@@ -117,18 +170,20 @@ describe('ConversionController', function () {
beforeEach(async function (ctx) {
ctx.req = {
file: { path: '/path/to/uploaded/file.docx' },
query: { type: 'docx' },
}
await ctx.ConversionController.convertDocxToLaTeX(ctx.req, ctx.res)
await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
})
it('should call the conversion manager with the uploaded file path', function (ctx) {
it('should call the conversion manager with the uploaded file path and type', function (ctx) {
sinon.assert.calledWith(
ctx.ConversionManager.promises.convertDocxToLaTeXWithLock,
ctx.ConversionManager.promises.convertToLaTeXWithLock,
sinon.match(
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/
),
ctx.req.file.path
ctx.req.file.path,
'docx'
)
})
@@ -160,6 +215,28 @@ describe('ConversionController', function () {
})
})
describe('with conversionType=markdown', function () {
beforeEach(async function (ctx) {
ctx.req = {
file: { path: '/path/to/uploaded/file.md' },
query: { type: 'markdown' },
}
await ctx.ConversionController.convertDocumentToLaTeX(ctx.req, ctx.res)
})
it('should call the conversion manager with the uploaded file path and markdown type', function (ctx) {
sinon.assert.calledWith(
ctx.ConversionManager.promises.convertToLaTeXWithLock,
sinon.match(
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/
),
ctx.req.file.path,
'markdown'
)
})
})
describe('unsuccessfully', function () {
describe('on streaming error', function () {
it('should propagate the error and still clean up', async function (ctx) {
@@ -169,10 +246,13 @@ describe('ConversionController', function () {
res.attachment = sinon.stub()
res.setHeader = sinon.stub()
const req = { file: { path: '/path/to/uploaded/file.docx' } }
const req = {
file: { path: '/path/to/uploaded/file.docx' },
query: { type: 'docx' },
}
await expect(
ctx.ConversionController.convertDocxToLaTeX(req, res)
ctx.ConversionController.convertDocumentToLaTeX(req, res)
).to.be.rejectedWith('mock stream error')
sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir)

416
services/clsi/test/unit/js/ConversionManager.test.js
View File

@@ -36,7 +36,6 @@ describe('ConversionManager', function () {
}
ctx.conversionId = 'test-conversion-id'
ctx.inputPath = '/path/to/input.docx'
ctx.conversionDir = '/compiles/test-conversion-id'
ctx.outputPath = '/compiles/test-conversion-id/output-uuid.zip'
@@ -65,188 +64,287 @@ describe('ConversionManager', function () {
ctx.uuidStub.restore()
})
describe('convertDocxToLaTeXWithLock', function () {
describe('general behavior', function () {
beforeEach(async function (ctx) {
ctx.result =
await ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
ctx.conversionId,
ctx.inputPath
describe('convertToLaTeXWithLock', function () {
describe('with conversionType=docx', function () {
beforeEach(function (ctx) {
ctx.inputPath = '/path/to/input.docx'
})
describe('file setup and pandoc args', function () {
beforeEach(async function (ctx) {
ctx.result =
await ctx.ConversionManager.promises.convertToLaTeXWithLock(
ctx.conversionId,
ctx.inputPath,
'docx'
)
})
it('should acquire a lock', async function (ctx) {
sinon.assert.calledWith(ctx.LockManager.acquire, ctx.conversionDir)
})
it('should copy the input file to the conversion directory with docx filename', async function (ctx) {
sinon.assert.calledWith(ctx.fs.mkdir, ctx.conversionDir, {
recursive: true,
})
sinon.assert.calledWith(
ctx.fs.copyFile,
ctx.inputPath,
Path.join(ctx.conversionDir, 'input.docx')
)
})
it('should acquire a lock', async function (ctx) {
sinon.assert.calledWith(ctx.LockManager.acquire, ctx.conversionDir)
})
it('should copy the input file to the conversion directory', async function (ctx) {
sinon.assert.calledWith(ctx.fs.mkdir, ctx.conversionDir, {
recursive: true,
})
sinon.assert.calledWith(
ctx.fs.copyFile,
ctx.inputPath,
Path.join(ctx.conversionDir, 'input.docx')
)
})
it('should convert conversion timeout to milliseconds', async function (ctx) {
expect(ctx.CommandRunner.promises.run.firstCall.args[4]).toBe(60_000)
expect(ctx.CommandRunner.promises.run.secondCall.args[4]).toBe(60_000)
})
it('should run pandoc followed by zip in the conversion directory', function (ctx) {
expect(ctx.CommandRunner.promises.run.callCount).toBe(2)
expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([
ctx.conversionId,
[
'pandoc',
'input.docx',
'--output',
'main.tex',
'--extract-media=.',
'--from',
'docx+citations',
'--to',
'latex',
'--citeproc',
'--standalone',
],
ctx.conversionDir,
ctx.Settings.pandocImage,
60_000,
{},
'conversions',
])
expect(ctx.CommandRunner.promises.run.secondCall.args).toEqual([
ctx.conversionId,
['zip', '-r', 'output-uuid.zip', '.'],
ctx.conversionDir,
ctx.Settings.pandocImage,
60_000,
{},
'conversions',
])
})
})
describe('successful conversion', function () {
beforeEach(async function (ctx) {
ctx.CommandRunner.promises.run.resolves({
stdout: 'mock-stdout',
stderr: 'mock-stderr',
exitCode: 0,
})
ctx.result =
await ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
it('should convert conversion timeout to milliseconds', async function (ctx) {
expect(ctx.CommandRunner.promises.run.firstCall.args[4]).toBe(60_000)
expect(ctx.CommandRunner.promises.run.secondCall.args[4]).toBe(60_000)
})
it('should run pandoc with docx args followed by zip', function (ctx) {
expect(ctx.CommandRunner.promises.run.callCount).toBe(2)
expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([
ctx.conversionId,
ctx.inputPath
)
})
it('should remove the source document after conversion', async function (ctx) {
sinon.assert.calledWith(
ctx.fs.unlink,
Path.join(ctx.conversionDir, 'input.docx')
)
})
it('should return the conversion directory', function (ctx) {
expect(ctx.result).toBe(ctx.outputPath)
})
it('should release the lock', function (ctx) {
sinon.assert.called(ctx.lock.release)
})
})
describe('unsuccessful conversion (exitcode)', function () {
beforeEach(async function (ctx) {
ctx.CommandRunner.promises.run.resolves({
stdout: 'mock-stdout',
stderr: 'mock-stderr',
exitCode: 63,
})
await expect(
ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
[
'pandoc',
'input.docx',
'--output',
'main.tex',
'--to',
'latex',
'--standalone',
'--extract-media=.',
'--from',
'docx+citations',
'--citeproc',
],
ctx.conversionDir,
ctx.Settings.pandocImage,
60_000,
{},
'conversions',
])
expect(ctx.CommandRunner.promises.run.secondCall.args).toEqual([
ctx.conversionId,
ctx.inputPath
)
).to.be.rejectedWith('pandoc conversion failed')
})
it('should remove the entire conversion directory', async function (ctx) {
sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
force: true,
recursive: true,
['zip', '-r', 'output-uuid.zip', '.'],
ctx.conversionDir,
ctx.Settings.pandocImage,
60_000,
{},
'conversions',
])
})
})
it('should release the lock', function (ctx) {
sinon.assert.called(ctx.lock.release)
})
})
describe('unsuccessful compression (exitcode)', function () {
beforeEach(async function (ctx) {
ctx.CommandRunner.promises.run
.onFirstCall()
.resolves({
stdout: 'mock-pandoc-stdout',
stderr: 'mock-pandoc-stderr',
describe('successful conversion', function () {
beforeEach(async function (ctx) {
ctx.CommandRunner.promises.run.resolves({
stdout: 'mock-stdout',
stderr: 'mock-stderr',
exitCode: 0,
})
.onSecondCall()
.resolves({
stdout: 'mock-zip-stdout',
stderr: 'mock-zip-stderr',
exitCode: 12,
})
await expect(
ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
ctx.conversionId,
ctx.inputPath
ctx.result =
await ctx.ConversionManager.promises.convertToLaTeXWithLock(
ctx.conversionId,
ctx.inputPath,
'docx'
)
})
it('should remove the source document after conversion', async function (ctx) {
sinon.assert.calledWith(
ctx.fs.unlink,
Path.join(ctx.conversionDir, 'input.docx')
)
).to.be.rejectedWith('pandoc conversion failed')
})
})
it('should remove the entire conversion directory', async function (ctx) {
sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
force: true,
recursive: true,
it('should return the output zip path', function (ctx) {
expect(ctx.result).toBe(ctx.outputPath)
})
it('should release the lock', function (ctx) {
sinon.assert.called(ctx.lock.release)
})
})
it('should release the lock', function (ctx) {
sinon.assert.called(ctx.lock.release)
describe('unsuccessful conversion (exitcode)', function () {
beforeEach(async function (ctx) {
ctx.CommandRunner.promises.run.resolves({
stdout: 'mock-stdout',
stderr: 'mock-stderr',
exitCode: 63,
})
await expect(
ctx.ConversionManager.promises.convertToLaTeXWithLock(
ctx.conversionId,
ctx.inputPath,
'docx'
)
).to.be.rejectedWith('pandoc conversion failed')
})
it('should remove the entire conversion directory', async function (ctx) {
sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
force: true,
recursive: true,
})
})
it('should release the lock', function (ctx) {
sinon.assert.called(ctx.lock.release)
})
})
describe('unsuccessful compression (exitcode)', function () {
beforeEach(async function (ctx) {
ctx.CommandRunner.promises.run
.onFirstCall()
.resolves({
stdout: 'mock-pandoc-stdout',
stderr: 'mock-pandoc-stderr',
exitCode: 0,
})
.onSecondCall()
.resolves({
stdout: 'mock-zip-stdout',
stderr: 'mock-zip-stderr',
exitCode: 12,
})
await expect(
ctx.ConversionManager.promises.convertToLaTeXWithLock(
ctx.conversionId,
ctx.inputPath,
'docx'
)
).to.be.rejectedWith('pandoc conversion failed')
})
it('should remove the entire conversion directory', async function (ctx) {
sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
force: true,
recursive: true,
})
})
it('should release the lock', function (ctx) {
sinon.assert.called(ctx.lock.release)
})
})
describe('unsuccessful conversion (throws)', function () {
beforeEach(async function (ctx) {
ctx.CommandRunner.promises.run.rejects(
new Error('mock conversion error')
)
await expect(
ctx.ConversionManager.promises.convertToLaTeXWithLock(
ctx.conversionId,
ctx.inputPath,
'docx'
)
).to.be.rejectedWith('pandoc conversion failed')
})
it('should remove the entire conversion directory', async function (ctx) {
sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
force: true,
recursive: true,
})
})
it('should release the lock', function (ctx) {
sinon.assert.called(ctx.lock.release)
})
})
})
describe('unsuccessful conversion (throws)', function () {
beforeEach(async function (ctx) {
ctx.CommandRunner.promises.run.rejects(
new Error('mock conversion error')
)
await expect(
ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
ctx.conversionId,
ctx.inputPath
)
).to.be.rejectedWith('pandoc conversion failed')
describe('with conversionType=markdown', function () {
beforeEach(function (ctx) {
ctx.inputPath = '/path/to/input.md'
})
it('should remove the entire conversion directory', async function (ctx) {
sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
force: true,
recursive: true,
describe('file setup and pandoc args', function () {
beforeEach(async function (ctx) {
ctx.result =
await ctx.ConversionManager.promises.convertToLaTeXWithLock(
ctx.conversionId,
ctx.inputPath,
'markdown'
)
})
it('should copy the input file to the conversion directory with md filename', async function (ctx) {
sinon.assert.calledWith(ctx.fs.mkdir, ctx.conversionDir, {
recursive: true,
})
sinon.assert.calledWith(
ctx.fs.copyFile,
ctx.inputPath,
Path.join(ctx.conversionDir, 'input.md')
)
})
it('should run pandoc with markdown args followed by zip', function (ctx) {
expect(ctx.CommandRunner.promises.run.callCount).toBe(2)
expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([
ctx.conversionId,
[
'pandoc',
'input.md',
'--output',
'main.tex',
'--to',
'latex',
'--standalone',
'--from',
'markdown',
],
ctx.conversionDir,
ctx.Settings.pandocImage,
60_000,
{},
'conversions',
])
expect(ctx.CommandRunner.promises.run.secondCall.args).toEqual([
ctx.conversionId,
['zip', '-r', 'output-uuid.zip', '.'],
ctx.conversionDir,
ctx.Settings.pandocImage,
60_000,
{},
'conversions',
])
})
})
it('should release the lock', function (ctx) {
sinon.assert.called(ctx.lock.release)
describe('successful conversion', function () {
beforeEach(async function (ctx) {
ctx.CommandRunner.promises.run.resolves({
stdout: 'mock-stdout',
stderr: 'mock-stderr',
exitCode: 0,
})
ctx.result =
await ctx.ConversionManager.promises.convertToLaTeXWithLock(
ctx.conversionId,
ctx.inputPath,
'markdown'
)
})
it('should remove the source document after conversion', async function (ctx) {
sinon.assert.calledWith(
ctx.fs.unlink,
Path.join(ctx.conversionDir, 'input.md')
)
})
it('should return the output zip path', function (ctx) {
expect(ctx.result).toBe(ctx.outputPath)
})
})
})
})

1
services/web/app/src/Features/Project/ProjectListController.mjs
View File

@@ -531,6 +531,7 @@ async function projectListPage(req, res, next) {
// Split tests that will be made available to the frontend
'import-docx',
'overleaf-library',
'import-markdown',
].filter(Boolean)
await Promise.all(

16
services/web/app/src/Features/Uploads/DocumentConversionManager.mjs
View File

@@ -11,20 +11,26 @@ import OError from '@overleaf/o-error'
import FormData from 'form-data'
import { FileTooLargeError } from '../Errors/Errors.js'
async function convertDocxToLaTeXZipArchive(path, userId) {
async function convertDocumentToLaTeXZipArchive(path, userId, conversionType) {
const clsiUrl = new URL(Settings.apis.clsi.url)
const limits = await CompileManager.promises._getUserCompileLimits(userId)
clsiUrl.pathname = '/convert/docx-to-latex'
// Uncomment this and remove the line below when the deploy is done.
// clsiUrl.pathname = '/convert/document-to-latex'
clsiUrl.pathname =
conversionType === 'docx'
? '/convert/docx-to-latex'
: '/convert/document-to-latex'
clsiUrl.searchParams.set('compileBackendClass', limits.compileBackendClass)
clsiUrl.searchParams.set('compileGroup', limits.compileGroup)
clsiUrl.searchParams.set('type', conversionType)
const formData = new FormData()
formData.append('qqfile', fs.createReadStream(path))
logger.debug(
{ clsiUrl: clsiUrl.toString() },
'sending docx to CLSI for conversion'
{ clsiUrl: clsiUrl.toString(), conversionType },
'sending document to CLSI for conversion'
)
const outputFileName = crypto.randomUUID() + '_document-conversion' + '.zip'
@@ -99,7 +105,7 @@ async function convertProjectToDocument(projectId, userId, type) {
export default {
promises: {
convertDocxToLaTeXZipArchive,
convertDocumentToLaTeXZipArchive,
convertProjectToDocument,
},
}

19
services/web/app/src/Features/Uploads/ProjectUploadController.mjs
View File

@@ -178,16 +178,21 @@ async function uploadFile(req, res, next) {
* @param {any} res
* @param {any} next
*/
async function importDocx(req, res, next) {
async function importDocument(req, res, next) {
const userId = SessionManager.getLoggedInUserId(req.session)
logger.debug({ path: req.file?.path, userId }, 'importing docx file')
const { path } = req.file
const name = Path.basename(req.body.name, '.docx')
const conversionType = req.query.type
if (!['docx', 'markdown'].includes(conversionType)) {
return res.status(400).json({ success: false, error: 'invalid_type' })
}
const name = Path.basename(req.body.name, Path.extname(req.body.name))
logger.debug({ path, userId, conversionType }, 'importing document file')
try {
const archivePath =
await DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
await DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
path,
userId
userId,
conversionType
)
try {
const project =
@@ -207,7 +212,7 @@ async function importDocx(req, res, next) {
})
}
} catch (error) {
logger.error({ error }, 'error importing docx file')
logger.error({ error }, 'error importing document file')
if (
error instanceof FileTooLargeError ||
error?.name === 'FileTooLargeError'
@@ -267,5 +272,5 @@ export default {
uploadProject,
uploadFile: expressify(uploadFile),
multerMiddleware,
importDocx: expressify(importDocx),
importDocument: expressify(importDocument),
}

14
services/web/app/src/Features/Uploads/UploadsRouter.mjs
View File

@@ -28,12 +28,24 @@ export default {
)
if (Settings.enablePandocConversions) {
webRouter.post(
'/project/new/import-document',
AuthenticationController.requireLogin(),
RateLimiterMiddleware.rateLimit(rateLimiters.projectUpload),
ProjectUploadController.multerMiddleware,
ProjectUploadController.importDocument
)
// Keep old route for backwards compatibility with old frontends that haven't reloaded
webRouter.post(
'/project/new/import-docx',
AuthenticationController.requireLogin(),
RateLimiterMiddleware.rateLimit(rateLimiters.projectUpload),
ProjectUploadController.multerMiddleware,
ProjectUploadController.importDocx
(req, res, next) => {
req.query.type = 'docx'
next()
},
ProjectUploadController.importDocument
)
}

5
services/web/frontend/extracted-translations.json
View File

@@ -287,6 +287,7 @@
"choose_a_custom_color": "",
"choose_from_group_members": "",
"choose_how_you_search_your_references": "",
"choose_markdown_file": "",
"choose_which_experiments": "",
"choose_word_document": "",
"citation": "",
@@ -905,12 +906,13 @@
"image_url": "",
"image_width": "",
"import_a_bibtex_file_from_your_provider_account": "",
"import_document_description": "",
"import_existing_projects_from_github": "",
"import_from_github": "",
"import_idp_metadata": "",
"import_markdown_file": "",
"import_to_sharelatex": "",
"import_word_document": "",
"import_word_document_description": "",
"imported_from_another_project_at_date": "",
"imported_from_external_provider_at_date": "",
"imported_from_mendeley_at_date": "",
@@ -1148,6 +1150,7 @@
"manager": "",
"managers_management": "",
"managing_your_subscription": "",
"markdown_import_feedback_message": "",
"marked_as_resolved": "",
"math": "",
"math_display": "",

4
services/web/frontend/js/features/ide-react/components/global-toasts.tsx
View File

@@ -6,7 +6,7 @@ import { debugConsole } from '@/utils/debugging'
import importOverleafModules from '../../../../macros/import-overleaf-module.macro'
import { OLToastContainer } from '@/shared/components/ol/ol-toast-container'
import clipboardToastGenerators from '@/features/source-editor/components/clipboard-toasts'
import importDocxFeedbackToastGenerators from '@/features/project-list/components/new-project-button/import-docx-feedback-toast'
import importDocumentFeedbackToastGenerators from '@/features/project-list/components/new-project-button/import-document-feedback-toast'
import exportDocumentToastGenerators from '@/features/ide-react/components/toolbar/export-document-toasts'
const moduleGeneratorsImport = importOverleafModules('toastGenerators') as {
@@ -29,7 +29,7 @@ type GlobalToastGenerator = (
const GENERATOR_LIST: GlobalToastGeneratorEntry[] = [
...moduleGenerators.flat(),
...clipboardToastGenerators,
...importDocxFeedbackToastGenerators,
...importDocumentFeedbackToastGenerators,
...exportDocumentToastGenerators,
]
const GENERATOR_MAP: Map<string, GlobalToastGenerator> = new Map(

4
services/web/frontend/js/features/ide-react/components/modals/modals.tsx
View File

@@ -3,7 +3,7 @@ import ForceDisconnected from '@/features/ide-react/components/modals/force-disc
import { UnsavedDocs } from '@/features/ide-react/components/unsaved-docs/unsaved-docs'
import SystemMessages from '@/shared/components/system-messages'
import ViewOnlyAccessModal from '@/features/share-project-modal/components/view-only-access-modal'
import ProjectConvertedFromDocxModal from '@/features/ide-react/components/modals/project-converted-from-docx-modal'
import ProjectConvertedFromDocumentModal from '@/features/ide-react/components/modals/project-converted-from-document-modal'
export const Modals = memo(() => {
return (
@@ -12,7 +12,7 @@ export const Modals = memo(() => {
<UnsavedDocs />
<SystemMessages />
<ViewOnlyAccessModal />
<ProjectConvertedFromDocxModal />
<ProjectConvertedFromDocumentModal />
</>
)
})

35
services/web/frontend/js/features/ide-react/components/modals/project-converted-from-docx-modal.tsx → services/web/frontend/js/features/ide-react/components/modals/project-converted-from-document-modal.tsx
View File

@@ -8,36 +8,35 @@ import {
} from '@/shared/components/ol/ol-modal'
import OLButton from '@/shared/components/ol/ol-button'
import { useEffect, useState } from 'react'
import { showImportDocxFeedbackToast } from '@/features/project-list/components/new-project-button/import-docx-feedback-toast'
import { showImportDocumentFeedbackToast } from '@/features/project-list/components/new-project-button/import-document-feedback-toast'
function ProjectConvertedFromDocxModal() {
const [
showProjectConvertedFromDocxModal,
setShowProjectConvertedFromDocxModal,
] = useState(false)
function ProjectConvertedFromDocumentModal() {
const [convertedFrom, setConvertedFrom] = useState<string | null>(null)
useEffect(() => {
const query = window.location.search
const queryString = new URLSearchParams(query)
const queryString = new URLSearchParams(window.location.search)
const from = queryString.get('converted-from')
if (queryString.get('converted-from-docx') === 'true') {
setShowProjectConvertedFromDocxModal(true)
if (from) {
setConvertedFrom(from)
// Clean the URL immediately so a refresh doesn't trigger the modal again,
// but preserve other search params and the hash.
const url = new URL(window.location.href)
url.searchParams.delete('converted-from-docx')
url.searchParams.delete('converted-from')
window.history.replaceState(window.history.state, '', url.toString())
}
}, [])
return (
<>
{showProjectConvertedFromDocxModal && (
<ProjectConvertedFromDocxModalContent
{convertedFrom && (
<ProjectConvertedFromImportModalContent
onHide={() => {
setShowProjectConvertedFromDocxModal(false)
showImportDocxFeedbackToast()
setConvertedFrom(null)
if (convertedFrom === 'docx' || convertedFrom === 'markdown') {
showImportDocumentFeedbackToast(convertedFrom)
}
}}
/>
)}
@@ -45,7 +44,7 @@ function ProjectConvertedFromDocxModal() {
)
}
function ProjectConvertedFromDocxModalContent({
function ProjectConvertedFromImportModalContent({
onHide,
}: {
onHide: () => void
@@ -57,7 +56,7 @@ function ProjectConvertedFromDocxModalContent({
show
animation
onHide={onHide}
id="converted-from-docx-modal"
id="converted-from-document-modal"
backdrop="static"
>
<OLModalHeader>
@@ -73,4 +72,4 @@ function ProjectConvertedFromDocxModalContent({
)
}
export default ProjectConvertedFromDocxModal
export default ProjectConvertedFromDocumentModal

18
services/web/frontend/js/features/project-list/components/new-project-button.tsx
View File

@@ -62,6 +62,9 @@ function NewProjectButton({
const docxImportEnabled =
useFeatureFlag('import-docx') &&
getMeta('ol-ExposedSettings').enablePandocConversions
const markdownImportEnabled =
useFeatureFlag('import-markdown') &&
getMeta('ol-ExposedSettings').enablePandocConversions
const sendTrackingEvent = useCallback(
({
dropdownMenu,
@@ -228,6 +231,21 @@ function NewProjectButton({
</DropdownItem>
</li>
)}
{markdownImportEnabled && (
<li role="none">
<DropdownItem
onClick={e =>
handleModalMenuClick(e, {
modalVariant: 'import_markdown',
dropdownMenuEvent: 'import-markdown',
})
}
trailingIcon={<MaterialIcon type="fiber_new" />}
>
{t('import_markdown_file')}
</DropdownItem>
</li>
)}
<li role="none">
{ImportProjectFromGithubMenu && (
<ImportProjectFromGithubMenu

67
services/web/frontend/js/features/project-list/components/new-project-button/import-document-feedback-toast.tsx Normal file
View File

@@ -0,0 +1,67 @@
import { GlobalToastGeneratorEntry } from '@/features/ide-react/components/global-toasts'
import { Trans } from 'react-i18next'
const DocxImportFeedbackToast = () => (
<div>
<Trans
i18nKey="docx_import_feedback_message"
components={[
/* eslint-disable-next-line jsx-a11y/anchor-has-content, react/jsx-key */
<a
href="https://forms.gle/B1qrdiD983YcQCJA9"
target="_blank"
rel="noopener noreferrer"
/>,
]}
/>
</div>
)
const MarkdownImportFeedbackToast = () => (
<div>
<Trans
i18nKey="markdown_import_feedback_message"
components={[
/* eslint-disable-next-line jsx-a11y/anchor-has-content, react/jsx-key */
<a
href="https://forms.gle/B1qrdiD983YcQCJA9"
target="_blank"
rel="noopener noreferrer"
/>,
]}
/>
</div>
)
const generators: GlobalToastGeneratorEntry[] = [
{
key: 'import:docx-feedback',
generator: () => ({
content: <DocxImportFeedbackToast />,
type: 'info',
autoHide: false,
isDismissible: true,
}),
},
{
key: 'import:markdown-feedback',
generator: () => ({
content: <MarkdownImportFeedbackToast />,
type: 'info',
autoHide: false,
isDismissible: true,
}),
},
]
export default generators
export const showImportDocumentFeedbackToast = (type: 'docx' | 'markdown') => {
const key =
type === 'markdown' ? 'import:markdown-feedback' : 'import:docx-feedback'
window.dispatchEvent(
new CustomEvent('ide:show-toast', {
detail: { key },
})
)
}

44
services/web/frontend/js/features/project-list/components/new-project-button/import-docx-modal.tsx → services/web/frontend/js/features/project-list/components/new-project-button/import-document-modal.tsx
View File

@@ -1,3 +1,4 @@
import { useMemo } from 'react'
import { Dashboard } from '@uppy/react'
import { useTranslation } from 'react-i18next'
import { useProjectUploader } from '../../hooks/use-project-uploader'
@@ -13,19 +14,39 @@ import '@uppy/core/dist/style.css'
import '@uppy/dashboard/dist/style.css'
import BetaBadgeIcon from '@/shared/components/beta-badge-icon'
function ImportDocxModal({
function ImportDocumentModal({
type,
onHide,
openProject,
}: {
type: 'docx' | 'markdown'
onHide: () => void
openProject: (id: string, isConvertedFromDocx?: boolean) => void
openProject: (id: string, convertedFrom?: string) => void
}) {
const { t } = useTranslation()
const IMPORT_CONFIGS = useMemo(
() => ({
docx: {
allowedFileTypes: ['.docx'],
title: t('choose_word_document'),
browseLabel: 'Select .docx file',
dragLabel: '%{browseFiles} or \n\n Drag .docx file',
},
markdown: {
allowedFileTypes: ['.md'],
title: t('choose_markdown_file'),
browseLabel: 'Select .md file',
dragLabel: '%{browseFiles} or \n\n Drag .md file',
},
}),
[t]
)
const config = IMPORT_CONFIGS[type]
const uppy = useProjectUploader({
endpoint: '/project/new/import-docx',
allowedFileTypes: ['.docx'],
onSuccess: (projectId: string) => openProject(projectId, true),
endpoint: `/project/new/import-document?type=${type}`,
allowedFileTypes: config.allowedFileTypes,
onSuccess: (projectId: string) => openProject(projectId, type),
})
return (
@@ -36,16 +57,17 @@ function ImportDocxModal({
id="upload-project-modal"
backdrop="static"
>
{/* TODO: make necessary changes here for import document modal */}
<OLModalHeader>
<OLModalTitle as="h3" className="import-docx-modal-title">
{t('choose_word_document')}
<OLModalTitle as="h3" className="import-document-modal-title">
{config.title}
<span className="beta-icon-wrapper">
<BetaBadgeIcon />
</span>
</OLModalTitle>
</OLModalHeader>
<OLModalBody>
<p>{t('import_word_document_description')}</p>
<p>{t('import_document_description')}</p>
<Dashboard
uppy={uppy}
proudlyDisplayPoweredByUppy={false}
@@ -55,8 +77,8 @@ function ImportDocxModal({
height={300}
locale={{
strings: {
browseFiles: 'Select .docx file',
dropPasteFiles: '%{browseFiles} or \n\n Drag .docx file',
browseFiles: config.browseLabel,
dropPasteFiles: config.dragLabel,
},
}}
className="project-list-upload-project-modal-uppy-dashboard"
@@ -71,4 +93,4 @@ function ImportDocxModal({
)
}
export default ImportDocxModal
export default ImportDocumentModal

42
services/web/frontend/js/features/project-list/components/new-project-button/import-docx-feedback-toast.tsx
View File

@@ -1,42 +0,0 @@
import { GlobalToastGeneratorEntry } from '@/features/ide-react/components/global-toasts'
import { Trans } from 'react-i18next'
const ImportDocxFeedbackToast = () => {
return (
<div>
<Trans
i18nKey="docx_import_feedback_message"
components={[
/* eslint-disable-next-line jsx-a11y/anchor-has-content, react/jsx-key */
<a
href="https://forms.gle/B1qrdiD983YcQCJA9"
target="_blank"
rel="noopener noreferrer"
/>,
]}
/>
</div>
)
}
const generators: GlobalToastGeneratorEntry[] = [
{
key: 'import:docx-feedback',
generator: () => ({
content: <ImportDocxFeedbackToast />,
type: 'info',
autoHide: false,
isDismissible: true,
}),
},
]
export default generators
export const showImportDocxFeedbackToast = () => {
window.dispatchEvent(
new CustomEvent('ide:show-toast', {
detail: { key: 'import:docx-feedback' },
})
)
}

25
services/web/frontend/js/features/project-list/components/new-project-button/new-project-button-modal.tsx
View File

@@ -7,7 +7,7 @@ import { FullSizeLoadingSpinner } from '@/shared/components/loading-spinner'
import { useLocation } from '@/shared/hooks/use-location'
const UploadProjectModal = lazy(() => import('./upload-project-modal'))
const ImportDocxModal = lazy(() => import('./import-docx-modal'))
const ImportDocumentModal = lazy(() => import('./import-document-modal'))
export type NewProjectButtonModalVariant =
| 'blank_project'
@@ -15,6 +15,7 @@ export type NewProjectButtonModalVariant =
| 'upload_project'
| 'import_from_github'
| 'import_docx'
| 'import_markdown'
type NewProjectButtonModalProps = {
modal: Nullable<NewProjectButtonModalVariant>
@@ -32,9 +33,9 @@ function NewProjectButtonModal({ modal, onHide }: NewProjectButtonModalProps) {
const location = useLocation()
const openProject = useCallback(
(projectId: string, isConvertedFromDocx: boolean = false) => {
const url = isConvertedFromDocx
? `/project/${projectId}?converted-from-docx=true`
(projectId: string, convertedFrom?: string) => {
const url = convertedFrom
? `/project/${projectId}?converted-from=${convertedFrom}`
: `/project/${projectId}`
location.assign(url)
@@ -56,7 +57,21 @@ function NewProjectButtonModal({ modal, onHide }: NewProjectButtonModalProps) {
case 'import_docx':
return (
<Suspense fallback={<FullSizeLoadingSpinner delay={500} />}>
<ImportDocxModal onHide={onHide} openProject={openProject} />
<ImportDocumentModal
type="docx"
onHide={onHide}
openProject={openProject}
/>
</Suspense>
)
case 'import_markdown':
return (
<Suspense fallback={<FullSizeLoadingSpinner delay={500} />}>
<ImportDocumentModal
type="markdown"
onHide={onHide}
openProject={openProject}
/>
</Suspense>
)
case 'import_from_github':

17
services/web/frontend/js/features/project-list/components/welcome-message-new/welcome-message-create-new-project-dropdown.tsx
View File

@@ -64,6 +64,9 @@ function WelcomeMessageCreateNewProjectDropdown({
const docxImportEnabled =
useFeatureFlag('import-docx') &&
getMeta('ol-ExposedSettings').enablePandocConversions
const markdownImportEnabled =
useFeatureFlag('import-markdown') &&
getMeta('ol-ExposedSettings').enablePandocConversions
const { isOverleaf } = getMeta('ol-ExposedSettings')
@@ -153,6 +156,20 @@ function WelcomeMessageCreateNewProjectDropdown({
</DropdownItem>
</li>
)}
{markdownImportEnabled && (
<li role="none">
<DropdownItem
as="button"
onClick={e =>
handleDropdownItemClick(e, 'import_markdown', 'import-markdown')
}
tabIndex={-1}
trailingIcon={<MaterialIcon type="fiber_new" />}
>
{t('import_markdown_file')}
</DropdownItem>
</li>
)}
{isOverleaf && (
<li role="none">
<DropdownItem

2
services/web/frontend/stylesheets/pages/project-list.scss
View File

@@ -585,7 +585,7 @@ ul.project-list-filters {
}
}
.import-docx-modal-title {
.import-document-modal-title {
display: flex;
align-items: center;

5
services/web/locales/en.json
View File

@@ -379,6 +379,7 @@
"choose_a_new_password": "Choose a new password",
"choose_from_group_members": "Choose from group members",
"choose_how_you_search_your_references": "Choose how you search your references",
"choose_markdown_file": "Choose Markdown file",
"choose_which_experiments": "Choose which experiments you’d like to try.",
"choose_word_document": "Choose Word document",
"choose_your_plan": "Choose your plan",
@@ -1174,12 +1175,13 @@
"image_url": "Image URL",
"image_width": "Image width",
"import_a_bibtex_file_from_your_provider_account": "Import a BibTeX file from your __provider__ account",
"import_document_description": "Content will be imported from the selected document. Formatting may not be reproduced exactly.",
"import_existing_projects_from_github": "Import existing projects from GitHub",
"import_from_github": "Import from GitHub",
"import_idp_metadata": "Import IdP metadata",
"import_markdown_file": "Import Markdown file",
"import_to_sharelatex": "Import to __appName__",
"import_word_document": "Import Word document",
"import_word_document_description": "Content will be imported from the selected document. Formatting may not be reproduced exactly.",
"important_message": "Important message",
"imported_from_another_project_at_date": "Imported from <0>Another project</0>/__sourceEntityPathHTML__, at __formattedDate__ __relativeDate__",
"imported_from_external_provider_at_date": "Imported from <0>__shortenedUrlHTML__</0> at __formattedDate__ __relativeDate__",
@@ -1517,6 +1519,7 @@
"managers_management": "Managers management",
"managing_your_subscription": "Managing your subscription",
"march": "March",
"markdown_import_feedback_message": "Importing Markdown files is a new feature. <0>Let us know what you think</0>",
"marked_as_resolved": "Marked as resolved",
"math": "Math",
"math_display": "Math Display",

90
services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs
View File

@@ -88,12 +88,74 @@ describe('DocumentConversionManager', function () {
ctx.DocumentConversionManager = (await import(MODULE_PATH)).default
})
describe('convertDocxToLaTeXZipArchive', function () {
describe('successfully', function () {
describe('convertDocumentToLaTeXZipArchive', function () {
describe('with conversionType=docx', function () {
describe('successfully', function () {
beforeEach(async function (ctx) {
ctx.path = '/path/to/input.docx'
ctx.userId = 'test-user-id'
ctx.response = {
headers: {
get: sinon.stub().returns(null),
},
}
ctx.response.headers.get.withArgs('Content-Length').returns('50')
ctx.fetchUtils.fetchStreamWithResponse.resolves({
stream: 'mocked-fetch-stream',
response: ctx.response,
})
ctx.result =
await ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
ctx.path,
ctx.userId,
'docx'
)
})
it('should call fetchStreamWithResponse with the correct URL and form data', function (ctx) {
const expectedUrl = new URL(ctx.Settings.apis.clsi.url)
// TODO: revert this to '/convert/document-to-latex' once the deploy is done (PR #32857)
expectedUrl.pathname = '/convert/docx-to-latex'
expectedUrl.searchParams.set(
'compileBackendClass',
'test-backend-class'
)
expectedUrl.searchParams.set('compileGroup', 'test-compile-group')
expectedUrl.searchParams.set('type', 'docx')
sinon.assert.calledWith(
ctx.fetchUtils.fetchStreamWithResponse,
sinon.match(url => url.toString() === expectedUrl.toString()),
{
method: 'POST',
body: sinon.match.instanceOf(FormData),
signal: sinon.match.instanceOf(AbortSignal),
}
)
})
it('should pipe result into the output file', function (ctx) {
sinon.assert.calledWith(
ctx.nodeStream.pipeline,
'mocked-fetch-stream',
'mocked-write-stream'
)
})
it('should return a path to the output file', function (ctx) {
expect(ctx.result).to.match(
/\/path\/to\/dump\/folder\/[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}_document-conversion\.zip/
)
})
})
})
describe('with conversionType=markdown', function () {
beforeEach(async function (ctx) {
ctx.path = '/path/to/input.docx'
ctx.path = '/path/to/input.md'
ctx.userId = 'test-user-id'
ctx.outputPath = '/path/to/output.zip'
ctx.response = {
headers: {
get: sinon.stub().returns(null),
@@ -107,20 +169,22 @@ describe('DocumentConversionManager', function () {
})
ctx.result =
await ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
await ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
ctx.path,
ctx.userId
ctx.userId,
'markdown'
)
})
it('should call fetchStreamWithResponse with the correct URL and form data', function (ctx) {
it('should call fetchStreamWithResponse with the correct URL including markdown type', function (ctx) {
const expectedUrl = new URL(ctx.Settings.apis.clsi.url)
expectedUrl.pathname = '/convert/docx-to-latex'
expectedUrl.pathname = '/convert/document-to-latex'
expectedUrl.searchParams.set(
'compileBackendClass',
'test-backend-class'
)
expectedUrl.searchParams.set('compileGroup', 'test-compile-group')
expectedUrl.searchParams.set('type', 'markdown')
sinon.assert.calledWith(
ctx.fetchUtils.fetchStreamWithResponse,
@@ -158,9 +222,10 @@ describe('DocumentConversionManager', function () {
)
await expect(
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
ctx.path,
ctx.userId
ctx.userId,
'docx'
)
).to.be.rejectedWith('document conversion failed')
})
@@ -195,9 +260,10 @@ describe('DocumentConversionManager', function () {
})
await expect(
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
ctx.path,
ctx.userId
ctx.userId,
'docx'
)
).to.be.rejectedWith(sinon.match.instanceOf(FileTooLargeError))
})

118
services/web/test/unit/src/Uploads/ProjectUploadController.test.mjs
View File

@@ -48,7 +48,7 @@ describe('ProjectUploadController', function () {
}
ctx.DocumentConversionManager = {
promises: {
convertDocxToLaTeXZipArchive: sinon.stub(),
convertDocumentToLaTeXZipArchive: sinon.stub(),
},
}
@@ -463,7 +463,7 @@ describe('ProjectUploadController', function () {
})
})
describe('importDocx', function () {
describe('importDocument', function () {
beforeEach(async function (ctx) {
ctx.req.file = {
path: '/path/to/uploaded/file.docx',
@@ -471,13 +471,73 @@ describe('ProjectUploadController', function () {
ctx.req.body = {
name: 'file.docx',
}
ctx.req.query = { type: 'docx' }
ctx.archivePath = '/path/to/archive.zip'
ctx.fsPromises.unlink = sinon.stub().resolves()
})
describe('successfully', async function () {
describe('with conversionType=docx', async function () {
describe('successfully', async function () {
beforeEach(async function (ctx) {
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
sinon.stub().resolves(ctx.archivePath)
ctx.ProjectUploadManager.promises.createProjectFromZipArchive = sinon
.stub()
.resolves({
_id: 'new-project-id',
})
await new Promise(resolve => {
ctx.res.json = data => {
expect(data.success).to.be.true
expect(data.project_id).to.equal('new-project-id')
resolve()
}
ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
})
})
it('should call the DocumentConversionManager with file path and type', function (ctx) {
expect(
ctx.DocumentConversionManager.promises
.convertDocumentToLaTeXZipArchive
).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'docx')
})
it('should use the resulting archive to create a new project', function (ctx) {
expect(
ctx.ProjectUploadManager.promises.createProjectFromZipArchive
).to.have.been.calledWith(ctx.user_id, 'file', ctx.archivePath)
})
it('should set the compiler to lualatex', function (ctx) {
expect(
ctx.ProjectOptionsHandler.promises.setCompiler
).to.have.been.calledWith('new-project-id', 'lualatex')
})
it('should unlink the archive after creating the project', function (ctx) {
expect(ctx.fsPromises.unlink).to.have.been.calledWith(ctx.archivePath)
})
it('should unlink the uploaded file', function (ctx) {
expect(ctx.fsPromises.unlink).to.have.been.calledWith(
ctx.req.file.path
)
})
})
})
describe('with conversionType=markdown', async function () {
beforeEach(async function (ctx) {
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive =
ctx.req.file = {
path: '/path/to/uploaded/file.md',
}
ctx.req.body = {
name: 'file.md',
}
ctx.req.query = { type: 'markdown' }
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
sinon.stub().resolves(ctx.archivePath)
ctx.ProjectUploadManager.promises.createProjectFromZipArchive = sinon
.stub()
@@ -491,14 +551,15 @@ describe('ProjectUploadController', function () {
expect(data.project_id).to.equal('new-project-id')
resolve()
}
ctx.ProjectUploadController.importDocx(ctx.req, ctx.res)
ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
})
})
it('should call the DocumentConversionManager to convert the file', function (ctx) {
it('should call the DocumentConversionManager with file path and markdown type', function (ctx) {
expect(
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive
).to.have.been.calledWith(ctx.req.file.path, ctx.user_id)
ctx.DocumentConversionManager.promises
.convertDocumentToLaTeXZipArchive
).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'markdown')
})
it('should use the resulting archive to create a new project', function (ctx) {
@@ -522,9 +583,37 @@ describe('ProjectUploadController', function () {
})
})
describe('with an invalid conversionType', async function () {
beforeEach(async function (ctx) {
ctx.req.query = { type: 'invalid' }
await new Promise(resolve => {
ctx.res.json = data => {
expect(data).to.deep.equal({
success: false,
error: 'invalid_type',
})
resolve()
}
ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
})
})
it('should return http 400', function (ctx) {
expect(ctx.res.statusCode).to.equal(400)
})
it('should not call DocumentConversionManager', function (ctx) {
expect(
ctx.DocumentConversionManager.promises
.convertDocumentToLaTeXZipArchive
).not.to.have.been.called
})
})
describe('unsuccessfully', async function () {
beforeEach(async function (ctx) {
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive =
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
sinon.stub().rejects(new Error('Conversion failed'))
await new Promise(resolve => {
@@ -532,14 +621,15 @@ describe('ProjectUploadController', function () {
expect(data.success).to.be.false
resolve()
}
ctx.ProjectUploadController.importDocx(ctx.req, ctx.res)
ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
})
})
it('should call the DocumentConversionManager to convert the file', function (ctx) {
expect(
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive
).to.have.been.calledWith(ctx.req.file.path, ctx.user_id)
ctx.DocumentConversionManager.promises
.convertDocumentToLaTeXZipArchive
).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'docx')
})
it('should unlink the uploaded file', function (ctx) {
@@ -553,7 +643,7 @@ describe('ProjectUploadController', function () {
describe('when the converted archive is too large', async function () {
beforeEach(async function (ctx) {
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive =
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
sinon.stub().rejects(new FileTooLargeError('file too large'))
await new Promise(resolve => {
@@ -564,7 +654,7 @@ describe('ProjectUploadController', function () {
})
resolve()
}
ctx.ProjectUploadController.importDocx(ctx.req, ctx.res)
ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
})
})

2
services/web/types/assets.d.ts vendored
View File

@@ -27,3 +27,5 @@ declare module '*.txt' {
const src: string
export default src
}
declare module '*.css' {}