mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-05-23 17:19:37 +02:00
Merge pull request #29841 from overleaf/ar-convert-filestore-to-esm
[filestore] convert to ES modules GitOrigin-RevId: 404905973548bb6e437fff66b368e87be8249b73
This commit is contained in:
3
package-lock.json
generated
3
package-lock.json
generated
@@ -55300,7 +55300,8 @@
|
|||||||
"sinon": "9.0.2",
|
"sinon": "9.0.2",
|
||||||
"sinon-chai": "^3.7.0",
|
"sinon-chai": "^3.7.0",
|
||||||
"streamifier": "^0.1.1",
|
"streamifier": "^0.1.1",
|
||||||
"typescript": "^5.0.4"
|
"typescript": "^5.0.4",
|
||||||
|
"vitest": "^3.2.4"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"services/filestore/node_modules/diff": {
|
"services/filestore/node_modules/diff": {
|
||||||
|
|||||||
@@ -1,20 +1,17 @@
|
|||||||
// Metrics must be initialized before importing anything else
|
// Metrics must be initialized before importing anything else
|
||||||
require('@overleaf/metrics/initialize')
|
import '@overleaf/metrics/initialize.js'
|
||||||
|
|
||||||
const Events = require('node:events')
|
import Events from 'node:events'
|
||||||
const Metrics = require('@overleaf/metrics')
|
import Metrics from '@overleaf/metrics'
|
||||||
|
import logger from '@overleaf/logger'
|
||||||
|
import settings from '@overleaf/settings'
|
||||||
|
import express from 'express'
|
||||||
|
import fileController from './app/js/FileController.js'
|
||||||
|
import keyBuilder from './app/js/KeyBuilder.js'
|
||||||
|
import RequestLogger from './app/js/RequestLogger.js'
|
||||||
|
|
||||||
const logger = require('@overleaf/logger')
|
|
||||||
logger.initialize(process.env.METRICS_APP_NAME || 'filestore')
|
logger.initialize(process.env.METRICS_APP_NAME || 'filestore')
|
||||||
|
|
||||||
const settings = require('@overleaf/settings')
|
|
||||||
const express = require('express')
|
|
||||||
|
|
||||||
const fileController = require('./app/js/FileController')
|
|
||||||
const keyBuilder = require('./app/js/KeyBuilder')
|
|
||||||
|
|
||||||
const RequestLogger = require('./app/js/RequestLogger')
|
|
||||||
|
|
||||||
Events.setMaxListeners(20)
|
Events.setMaxListeners(20)
|
||||||
|
|
||||||
const app = express()
|
const app = express()
|
||||||
@@ -106,7 +103,7 @@ const port = settings.internal.filestore.port || 3009
|
|||||||
const host = settings.internal.filestore.host || '0.0.0.0'
|
const host = settings.internal.filestore.host || '0.0.0.0'
|
||||||
|
|
||||||
let server = null
|
let server = null
|
||||||
if (!module.parent) {
|
if (import.meta.main) {
|
||||||
// Called directly
|
// Called directly
|
||||||
server = app.listen(port, host, error => {
|
server = app.listen(port, host, error => {
|
||||||
if (error) {
|
if (error) {
|
||||||
@@ -157,4 +154,4 @@ function handleShutdownSignal(signal) {
|
|||||||
|
|
||||||
process.on('SIGTERM', handleShutdownSignal)
|
process.on('SIGTERM', handleShutdownSignal)
|
||||||
|
|
||||||
module.exports = app
|
export default app
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
const OError = require('@overleaf/o-error')
|
import OError from '@overleaf/o-error'
|
||||||
const { Errors } = require('@overleaf/object-persistor')
|
|
||||||
|
import { Errors } from '@overleaf/object-persistor'
|
||||||
|
|
||||||
class HealthCheckError extends OError {}
|
class HealthCheckError extends OError {}
|
||||||
class ConversionsDisabledError extends OError {}
|
class ConversionsDisabledError extends OError {}
|
||||||
@@ -19,12 +20,12 @@ class FailedCommandError extends OError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
export default {
|
||||||
FailedCommandError,
|
...Errors,
|
||||||
|
HealthCheckError,
|
||||||
ConversionsDisabledError,
|
ConversionsDisabledError,
|
||||||
ConversionError,
|
ConversionError,
|
||||||
HealthCheckError,
|
|
||||||
TimeoutError,
|
TimeoutError,
|
||||||
InvalidParametersError,
|
InvalidParametersError,
|
||||||
...Errors,
|
FailedCommandError,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
const FileHandler = require('./FileHandler')
|
import FileHandler from './FileHandler.js'
|
||||||
const metrics = require('@overleaf/metrics')
|
import metrics from '@overleaf/metrics'
|
||||||
const parseRange = require('range-parser')
|
import parseRange from 'range-parser'
|
||||||
const Errors = require('./Errors')
|
import Errors from './Errors.js'
|
||||||
const { pipeline } = require('node:stream')
|
import { pipeline } from 'node:stream'
|
||||||
|
|
||||||
const maxSizeInBytes = 1024 * 1024 * 1024 // 1GB
|
const maxSizeInBytes = 1024 * 1024 * 1024 // 1GB
|
||||||
|
|
||||||
module.exports = {
|
export default {
|
||||||
getFile,
|
getFile,
|
||||||
getFileHead,
|
getFileHead,
|
||||||
insertFile,
|
insertFile,
|
||||||
|
|||||||
@@ -1,15 +1,16 @@
|
|||||||
const metrics = require('@overleaf/metrics')
|
import metrics from '@overleaf/metrics'
|
||||||
const Settings = require('@overleaf/settings')
|
import Settings from '@overleaf/settings'
|
||||||
const { callbackify } = require('node:util')
|
import { callbackify } from 'node:util'
|
||||||
|
import SafeExec from './SafeExec.js'
|
||||||
|
import Errors from './Errors.js'
|
||||||
|
|
||||||
const safeExec = require('./SafeExec').promises
|
const { ConversionError } = Errors
|
||||||
const { ConversionError } = require('./Errors')
|
|
||||||
|
|
||||||
const APPROVED_FORMATS = ['png']
|
const APPROVED_FORMATS = ['png']
|
||||||
const FOURTY_SECONDS = 40 * 1000
|
const FOURTY_SECONDS = 40 * 1000
|
||||||
const KILL_SIGNAL = 'SIGTERM'
|
const KILL_SIGNAL = 'SIGTERM'
|
||||||
|
|
||||||
module.exports = {
|
export default {
|
||||||
convert: callbackify(convert),
|
convert: callbackify(convert),
|
||||||
thumbnail: callbackify(thumbnail),
|
thumbnail: callbackify(thumbnail),
|
||||||
preview: callbackify(preview),
|
preview: callbackify(preview),
|
||||||
@@ -81,7 +82,7 @@ async function _convert(sourcePath, requestedFormat, command) {
|
|||||||
command = Settings.commands.convertCommandPrefix.concat(command)
|
command = Settings.commands.convertCommandPrefix.concat(command)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await safeExec(command, {
|
await SafeExec.promises(command, {
|
||||||
killSignal: KILL_SIGNAL,
|
killSignal: KILL_SIGNAL,
|
||||||
timeout: FOURTY_SECONDS,
|
timeout: FOURTY_SECONDS,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,15 +1,17 @@
|
|||||||
const Settings = require('@overleaf/settings')
|
import Settings from '@overleaf/settings'
|
||||||
const { callbackify } = require('node:util')
|
import { callbackify } from 'node:util'
|
||||||
const fs = require('node:fs')
|
import fs from 'node:fs'
|
||||||
let PersistorManager = require('./PersistorManager')
|
import _PersistorManager from './PersistorManager.js'
|
||||||
const LocalFileWriter = require('./LocalFileWriter')
|
import LocalFileWriter from './LocalFileWriter.js'
|
||||||
const FileConverter = require('./FileConverter')
|
import FileConverter from './FileConverter.js'
|
||||||
const KeyBuilder = require('./KeyBuilder')
|
import KeyBuilder from './KeyBuilder.js'
|
||||||
const ImageOptimiser = require('./ImageOptimiser')
|
import ImageOptimiser from './ImageOptimiser.js'
|
||||||
const { ConversionError, InvalidParametersError } = require('./Errors')
|
import Errors from './Errors.js'
|
||||||
const metrics = require('@overleaf/metrics')
|
import metrics from '@overleaf/metrics'
|
||||||
|
|
||||||
module.exports = {
|
const { ConversionError, InvalidParametersError } = Errors
|
||||||
|
|
||||||
|
const FileHandler = {
|
||||||
insertFile: callbackify(insertFile),
|
insertFile: callbackify(insertFile),
|
||||||
getFile: callbackify(getFile),
|
getFile: callbackify(getFile),
|
||||||
getRedirectUrl: callbackify(getRedirectUrl),
|
getRedirectUrl: callbackify(getRedirectUrl),
|
||||||
@@ -22,8 +24,10 @@ module.exports = {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let PersistorManager = _PersistorManager
|
||||||
|
|
||||||
if (process.env.NODE_ENV === 'test') {
|
if (process.env.NODE_ENV === 'test') {
|
||||||
module.exports._TESTONLYSwapPersistorManager = _PersistorManager => {
|
FileHandler._TESTONLYSwapPersistorManager = _PersistorManager => {
|
||||||
PersistorManager = _PersistorManager
|
PersistorManager = _PersistorManager
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -183,3 +187,5 @@ async function _writeFileToDisk(bucket, key, opts) {
|
|||||||
const fileStream = await PersistorManager.getObjectStream(bucket, key, opts)
|
const fileStream = await PersistorManager.getObjectStream(bucket, key, opts)
|
||||||
return await LocalFileWriter.promises.writeStream(fileStream, key)
|
return await LocalFileWriter.promises.writeStream(fileStream, key)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export default FileHandler
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
const logger = require('@overleaf/logger')
|
import logger from '@overleaf/logger'
|
||||||
const metrics = require('@overleaf/metrics')
|
import metrics from '@overleaf/metrics'
|
||||||
const { callbackify } = require('node:util')
|
import { callbackify } from 'node:util'
|
||||||
const safeExec = require('./SafeExec').promises
|
import SafeExec from './SafeExec.js'
|
||||||
|
|
||||||
module.exports = {
|
export default {
|
||||||
compressPng: callbackify(compressPng),
|
compressPng: callbackify(compressPng),
|
||||||
promises: {
|
promises: {
|
||||||
compressPng,
|
compressPng,
|
||||||
@@ -19,7 +19,7 @@ async function compressPng(localPath, callback) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await safeExec(args, opts)
|
await SafeExec.promises(args, opts)
|
||||||
timer.done()
|
timer.done()
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err.code === 'SIGKILL') {
|
if (err.code === 'SIGKILL') {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
const settings = require('@overleaf/settings')
|
import settings from '@overleaf/settings'
|
||||||
const projectKey = require('./project_key')
|
import * as projectKey from './project_key.js'
|
||||||
|
|
||||||
module.exports = {
|
export default {
|
||||||
getConvertedFolderKey,
|
getConvertedFolderKey,
|
||||||
addCachingToKey,
|
addCachingToKey,
|
||||||
bucketFileKeyMiddleware,
|
bucketFileKeyMiddleware,
|
||||||
|
|||||||
@@ -1,13 +1,15 @@
|
|||||||
const fs = require('node:fs')
|
import fs from 'node:fs'
|
||||||
const crypto = require('node:crypto')
|
import crypto from 'node:crypto'
|
||||||
const path = require('node:path')
|
import path from 'node:path'
|
||||||
const Stream = require('node:stream')
|
import Stream from 'node:stream'
|
||||||
const { callbackify, promisify } = require('node:util')
|
import { callbackify, promisify } from 'node:util'
|
||||||
const metrics = require('@overleaf/metrics')
|
import metrics from '@overleaf/metrics'
|
||||||
const Settings = require('@overleaf/settings')
|
import Settings from '@overleaf/settings'
|
||||||
const { WriteError } = require('./Errors')
|
import Errors from './Errors.js'
|
||||||
|
|
||||||
module.exports = {
|
const { WriteError } = Errors
|
||||||
|
|
||||||
|
export default {
|
||||||
promises: {
|
promises: {
|
||||||
writeStream,
|
writeStream,
|
||||||
deleteFile,
|
deleteFile,
|
||||||
@@ -39,7 +41,7 @@ async function deleteFile(fsPath) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
await promisify(fs.unlink)(fsPath)
|
await fs.promises.unlink(fsPath)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err.code !== 'ENOENT') {
|
if (err.code !== 'ENOENT') {
|
||||||
throw new WriteError('failed to delete file', { fsPath }, err)
|
throw new WriteError('failed to delete file', { fsPath }, err)
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
const settings = require('@overleaf/settings')
|
import settings from '@overleaf/settings'
|
||||||
|
import ObjectPersistor from '@overleaf/object-persistor'
|
||||||
|
|
||||||
const persistorSettings = settings.filestore
|
const persistorSettings = settings.filestore
|
||||||
persistorSettings.paths = settings.path
|
persistorSettings.paths = settings.path
|
||||||
|
|
||||||
const ObjectPersistor = require('@overleaf/object-persistor')
|
|
||||||
const persistor = ObjectPersistor(persistorSettings)
|
const persistor = ObjectPersistor(persistorSettings)
|
||||||
|
|
||||||
module.exports = persistor
|
export default persistor
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
const logger = require('@overleaf/logger')
|
import logger from '@overleaf/logger'
|
||||||
const metrics = require('@overleaf/metrics')
|
import metrics from '@overleaf/metrics'
|
||||||
|
|
||||||
class RequestLogger {
|
class RequestLogger {
|
||||||
constructor() {
|
constructor() {
|
||||||
@@ -58,4 +58,4 @@ class RequestLogger {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = RequestLogger
|
export default RequestLogger
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
const lodashOnce = require('lodash.once')
|
import lodashOnce from 'lodash.once'
|
||||||
const childProcess = require('node:child_process')
|
import childProcess from 'node:child_process'
|
||||||
const Settings = require('@overleaf/settings')
|
import Settings from '@overleaf/settings'
|
||||||
const { ConversionsDisabledError, FailedCommandError } = require('./Errors')
|
import Errors from './Errors.js'
|
||||||
|
|
||||||
|
const { ConversionsDisabledError, FailedCommandError } = Errors
|
||||||
|
|
||||||
// execute a command in the same way as 'exec' but with a timeout that
|
// execute a command in the same way as 'exec' but with a timeout that
|
||||||
// kills all child processes
|
// kills all child processes
|
||||||
@@ -9,8 +11,9 @@ const { ConversionsDisabledError, FailedCommandError } = require('./Errors')
|
|||||||
// we spawn the command with 'detached:true' to make a new process
|
// we spawn the command with 'detached:true' to make a new process
|
||||||
// group, then we can kill everything in that process group.
|
// group, then we can kill everything in that process group.
|
||||||
|
|
||||||
module.exports = safeExec
|
export default safeExec
|
||||||
module.exports.promises = safeExecPromise
|
|
||||||
|
safeExec.promises = safeExecPromise
|
||||||
|
|
||||||
// options are {timeout: number-of-milliseconds, killSignal: signal-name}
|
// options are {timeout: number-of-milliseconds, killSignal: signal-name}
|
||||||
function safeExec(command, options, callback) {
|
function safeExec(command, options, callback) {
|
||||||
|
|||||||
@@ -1,23 +1,20 @@
|
|||||||
// Keep in sync with services/history-v1/storage/lib/project_key.js
|
// Keep in sync with services/history-v1/storage/lib/project_key.js
|
||||||
const path = require('node:path')
|
import path from 'node:path'
|
||||||
|
|
||||||
//
|
//
|
||||||
// The advice in http://docs.aws.amazon.com/AmazonS3/latest/dev/
|
// The advice in http://docs.aws.amazon.com/AmazonS3/latest/dev/
|
||||||
// request-rate-perf-considerations.html is to avoid sequential key prefixes,
|
// request-rate-perf-considerations.html is to avoid sequential key prefixes,
|
||||||
// so we reverse the project ID part of the key as they suggest.
|
// so we reverse the project ID part of the key as they suggest.
|
||||||
//
|
//
|
||||||
function format(projectId) {
|
export function format(projectId) {
|
||||||
const prefix = naiveReverse(pad(projectId))
|
const prefix = naiveReverse(pad(projectId))
|
||||||
return path.join(prefix.slice(0, 3), prefix.slice(3, 6), prefix.slice(6))
|
return path.join(prefix.slice(0, 3), prefix.slice(3, 6), prefix.slice(6))
|
||||||
}
|
}
|
||||||
|
|
||||||
function pad(number) {
|
export function pad(number) {
|
||||||
return (number || 0).toString().padStart(9, '0')
|
return (number || 0).toString().padStart(9, '0')
|
||||||
}
|
}
|
||||||
|
|
||||||
function naiveReverse(string) {
|
function naiveReverse(string) {
|
||||||
return string.split('').reverse().join('')
|
return string.split('').reverse().join('')
|
||||||
}
|
}
|
||||||
|
|
||||||
exports.format = format
|
|
||||||
exports.pad = pad
|
|
||||||
|
|||||||
@@ -8,4 +8,6 @@ filestore
|
|||||||
--pipeline-owner=🚉 Platform
|
--pipeline-owner=🚉 Platform
|
||||||
--public-repo=True
|
--public-repo=True
|
||||||
--test-acceptance-shards=SHARD_01_,SHARD_02_,SHARD_03_
|
--test-acceptance-shards=SHARD_01_,SHARD_02_,SHARD_03_
|
||||||
|
--test-unit-vitest=True
|
||||||
|
--tsconfig-extra-includes=vitest.config.unit.cjs,vitest.config.acceptance.cjs
|
||||||
--use-large-ci-runner=True
|
--use-large-ci-runner=True
|
||||||
|
|||||||
@@ -11,12 +11,14 @@ services:
|
|||||||
user: node
|
user: node
|
||||||
volumes:
|
volumes:
|
||||||
- ./reports:/overleaf/services/filestore/reports
|
- ./reports:/overleaf/services/filestore/reports
|
||||||
|
- ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
|
||||||
command: npm run test:unit:_run
|
command: npm run test:unit:_run
|
||||||
environment:
|
environment:
|
||||||
CI:
|
CI:
|
||||||
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
|
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
|
||||||
NODE_ENV: test
|
NODE_ENV: test
|
||||||
NODE_OPTIONS: "--unhandled-rejections=strict"
|
NODE_OPTIONS: "--unhandled-rejections=strict"
|
||||||
|
VITEST_NO_CACHE: true
|
||||||
|
|
||||||
|
|
||||||
test_acceptance:
|
test_acceptance:
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ services:
|
|||||||
- .:/overleaf/services/filestore
|
- .:/overleaf/services/filestore
|
||||||
- ../../node_modules:/overleaf/node_modules
|
- ../../node_modules:/overleaf/node_modules
|
||||||
- ../../libraries:/overleaf/libraries
|
- ../../libraries:/overleaf/libraries
|
||||||
|
- ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
|
||||||
working_dir: /overleaf/services/filestore
|
working_dir: /overleaf/services/filestore
|
||||||
environment:
|
environment:
|
||||||
MOCHA_GREP: ${MOCHA_GREP}
|
MOCHA_GREP: ${MOCHA_GREP}
|
||||||
|
|||||||
@@ -3,18 +3,19 @@
|
|||||||
"description": "An API for CRUD operations on binary files stored in S3",
|
"description": "An API for CRUD operations on binary files stored in S3",
|
||||||
"private": true,
|
"private": true,
|
||||||
"main": "app.js",
|
"main": "app.js",
|
||||||
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test:acceptance:run": "mocha --recursive --timeout 15000 $@ test/acceptance/js",
|
"test:acceptance:run": "mocha --recursive --timeout 15000 $@ test/acceptance/js",
|
||||||
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
|
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
|
||||||
"test:unit:run": "mocha --recursive $@ test/unit/js",
|
"test:unit:run": "mocha --recursive $@ test/unit/js",
|
||||||
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
|
"test:unit": "npm run test:unit:_run",
|
||||||
"start": "node app.js",
|
"start": "node app.js",
|
||||||
"nodemon": "node --watch app.js",
|
"nodemon": "node --watch app.js",
|
||||||
"lint": "eslint --max-warnings 0 --format unix .",
|
"lint": "eslint --max-warnings 0 --format unix .",
|
||||||
"format": "prettier --list-different $PWD/'**/{*.*js,*.ts}'",
|
"format": "prettier --list-different $PWD/'**/{*.*js,*.ts}'",
|
||||||
"format:fix": "prettier --write $PWD/'**/{*.*js,*.ts}'",
|
"format:fix": "prettier --write $PWD/'**/{*.*js,*.ts}'",
|
||||||
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
|
"test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js",
|
||||||
"test:unit:_run": "mocha --recursive --exit $@ test/unit/js",
|
"test:unit:_run": "vitest --config ./vitest.config.unit.cjs",
|
||||||
"lint:fix": "eslint --fix .",
|
"lint:fix": "eslint --fix .",
|
||||||
"types:check": "tsc --noEmit"
|
"types:check": "tsc --noEmit"
|
||||||
},
|
},
|
||||||
@@ -47,6 +48,7 @@
|
|||||||
"sinon": "9.0.2",
|
"sinon": "9.0.2",
|
||||||
"sinon-chai": "^3.7.0",
|
"sinon-chai": "^3.7.0",
|
||||||
"streamifier": "^0.1.1",
|
"streamifier": "^0.1.1",
|
||||||
"typescript": "^5.0.4"
|
"typescript": "^5.0.4",
|
||||||
|
"vitest": "^3.2.4"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
const ObjectPersistor = require('@overleaf/object-persistor')
|
import ObjectPersistor from '@overleaf/object-persistor'
|
||||||
const Settings = require('@overleaf/settings')
|
import Settings from '@overleaf/settings'
|
||||||
const { promisify } = require('node:util')
|
import { promisify } from 'node:util'
|
||||||
const App = require('../../../app')
|
import App from '../../../app.js'
|
||||||
const FileHandler = require('../../../app/js/FileHandler')
|
import FileHandler from '../../../app/js/FileHandler.js'
|
||||||
|
|
||||||
class FilestoreApp {
|
class FilestoreApp {
|
||||||
async runServer() {
|
async runServer() {
|
||||||
@@ -39,4 +39,4 @@ class FilestoreApp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = FilestoreApp
|
export default FilestoreApp
|
||||||
|
|||||||
@@ -1,19 +1,48 @@
|
|||||||
const chai = require('chai')
|
import chai from 'chai'
|
||||||
|
import fs from 'node:fs'
|
||||||
|
import Stream from 'node:stream'
|
||||||
|
import Settings from '@overleaf/settings'
|
||||||
|
import Path from 'node:path'
|
||||||
|
import FilestoreApp from './FilestoreApp.js'
|
||||||
|
import TestHelper from './TestHelper.js'
|
||||||
|
import fetch from 'node-fetch'
|
||||||
|
import { promisify } from 'node:util'
|
||||||
|
import { Storage } from '@google-cloud/storage'
|
||||||
|
import streamifier from 'streamifier'
|
||||||
|
import { ObjectId } from 'mongodb'
|
||||||
|
import { ListObjectsV2Command } from '@aws-sdk/client-s3'
|
||||||
|
import ChildProcess from 'node:child_process'
|
||||||
|
import chaiAsPromised from 'chai-as-promised'
|
||||||
|
|
||||||
|
// store settings for multiple backends, so that we can test each one.
|
||||||
|
// fs will always be available - add others if they are configured
|
||||||
|
import TestConfig from './TestConfig.js'
|
||||||
|
|
||||||
|
import {
|
||||||
|
AlreadyWrittenError,
|
||||||
|
NotFoundError,
|
||||||
|
NotImplementedError,
|
||||||
|
NoKEKMatchedError,
|
||||||
|
} from '@overleaf/object-persistor/src/Errors.js'
|
||||||
|
import {
|
||||||
|
PerProjectEncryptedS3Persistor,
|
||||||
|
RootKeyEncryptionKey,
|
||||||
|
} from '@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor.js'
|
||||||
|
import { S3Persistor } from '@overleaf/object-persistor/src/S3Persistor.js'
|
||||||
|
import crypto from 'node:crypto'
|
||||||
|
import { WritableBuffer } from '@overleaf/stream-utils'
|
||||||
|
import { gzipSync } from 'node:zlib'
|
||||||
|
|
||||||
const { expect } = chai
|
const { expect } = chai
|
||||||
const fs = require('node:fs')
|
|
||||||
const Stream = require('node:stream')
|
chai.use(chaiAsPromised)
|
||||||
const Settings = require('@overleaf/settings')
|
|
||||||
const Path = require('node:path')
|
const {
|
||||||
const FilestoreApp = require('./FilestoreApp')
|
BackendSettings,
|
||||||
const TestHelper = require('./TestHelper')
|
s3Config,
|
||||||
const fetch = require('node-fetch')
|
s3SSECConfig,
|
||||||
const { promisify } = require('node:util')
|
AWS_S3_USER_FILES_STORAGE_CLASS,
|
||||||
const { Storage } = require('@google-cloud/storage')
|
} = TestConfig
|
||||||
const streamifier = require('streamifier')
|
|
||||||
chai.use(require('chai-as-promised'))
|
|
||||||
const { ObjectId } = require('mongodb')
|
|
||||||
const ChildProcess = require('node:child_process')
|
|
||||||
const { ListObjectsV2Command } = require('@aws-sdk/client-s3')
|
|
||||||
|
|
||||||
const fsWriteFile = promisify(fs.writeFile)
|
const fsWriteFile = promisify(fs.writeFile)
|
||||||
const fsStat = promisify(fs.stat)
|
const fsStat = promisify(fs.stat)
|
||||||
@@ -30,29 +59,6 @@ process.on('unhandledRejection', e => {
|
|||||||
throw e
|
throw e
|
||||||
})
|
})
|
||||||
|
|
||||||
// store settings for multiple backends, so that we can test each one.
|
|
||||||
// fs will always be available - add others if they are configured
|
|
||||||
const {
|
|
||||||
BackendSettings,
|
|
||||||
s3Config,
|
|
||||||
s3SSECConfig,
|
|
||||||
AWS_S3_USER_FILES_STORAGE_CLASS,
|
|
||||||
} = require('./TestConfig')
|
|
||||||
const {
|
|
||||||
AlreadyWrittenError,
|
|
||||||
NotFoundError,
|
|
||||||
NotImplementedError,
|
|
||||||
NoKEKMatchedError,
|
|
||||||
} = require('@overleaf/object-persistor/src/Errors')
|
|
||||||
const {
|
|
||||||
PerProjectEncryptedS3Persistor,
|
|
||||||
RootKeyEncryptionKey,
|
|
||||||
} = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor')
|
|
||||||
const { S3Persistor } = require('@overleaf/object-persistor/src/S3Persistor')
|
|
||||||
const crypto = require('node:crypto')
|
|
||||||
const { WritableBuffer } = require('@overleaf/stream-utils')
|
|
||||||
const { gzipSync } = require('node:zlib')
|
|
||||||
|
|
||||||
describe('Filestore', function () {
|
describe('Filestore', function () {
|
||||||
this.timeout(1000 * 10)
|
this.timeout(1000 * 10)
|
||||||
const filestoreUrl = `http://127.0.0.1:${Settings.internal.filestore.port}`
|
const filestoreUrl = `http://127.0.0.1:${Settings.internal.filestore.port}`
|
||||||
@@ -899,7 +905,7 @@ describe('Filestore', function () {
|
|||||||
describe('with a pdf file', function () {
|
describe('with a pdf file', function () {
|
||||||
let localFileSize
|
let localFileSize
|
||||||
const localFileReadPath = Path.resolve(
|
const localFileReadPath = Path.resolve(
|
||||||
__dirname,
|
import.meta.dirname,
|
||||||
'../../fixtures/test.pdf'
|
'../../fixtures/test.pdf'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,7 @@
|
|||||||
const fs = require('node:fs')
|
import fs from 'node:fs'
|
||||||
const Path = require('node:path')
|
import Path from 'node:path'
|
||||||
const crypto = require('node:crypto')
|
import crypto from 'node:crypto'
|
||||||
const {
|
import { RootKeyEncryptionKey } from '@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor.js'
|
||||||
RootKeyEncryptionKey,
|
|
||||||
} = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor')
|
|
||||||
|
|
||||||
const AWS_S3_USER_FILES_STORAGE_CLASS =
|
const AWS_S3_USER_FILES_STORAGE_CLASS =
|
||||||
process.env.AWS_S3_USER_FILES_STORAGE_CLASS
|
process.env.AWS_S3_USER_FILES_STORAGE_CLASS
|
||||||
@@ -87,7 +85,10 @@ function gcsStores() {
|
|||||||
|
|
||||||
function fsStores() {
|
function fsStores() {
|
||||||
return {
|
return {
|
||||||
template_files: Path.resolve(__dirname, '../../../template_files'),
|
template_files: Path.resolve(
|
||||||
|
import.meta.dirname,
|
||||||
|
'../../../template_files'
|
||||||
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,7 +171,7 @@ function checkForUnexpectedTestFile() {
|
|||||||
'TestConfig.js',
|
'TestConfig.js',
|
||||||
'TestHelper.js',
|
'TestHelper.js',
|
||||||
]
|
]
|
||||||
for (const file of fs.readdirSync(__dirname).sort()) {
|
for (const file of fs.readdirSync(import.meta.dirname).sort()) {
|
||||||
if (!awareOfSharding.includes(file)) {
|
if (!awareOfSharding.includes(file)) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Found new test file ${file}: All tests must be aware of the SHARD_ prefix.`
|
`Found new test file ${file}: All tests must be aware of the SHARD_ prefix.`
|
||||||
@@ -180,7 +181,7 @@ function checkForUnexpectedTestFile() {
|
|||||||
}
|
}
|
||||||
checkForUnexpectedTestFile()
|
checkForUnexpectedTestFile()
|
||||||
|
|
||||||
module.exports = {
|
export default {
|
||||||
AWS_S3_USER_FILES_STORAGE_CLASS,
|
AWS_S3_USER_FILES_STORAGE_CLASS,
|
||||||
BackendSettings,
|
BackendSettings,
|
||||||
s3Config,
|
s3Config,
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
const streamifier = require('streamifier')
|
import streamifier from 'streamifier'
|
||||||
const fetch = require('node-fetch')
|
import fetch from 'node-fetch'
|
||||||
const ObjectPersistor = require('@overleaf/object-persistor')
|
import ObjectPersistor from '@overleaf/object-persistor'
|
||||||
|
import { expect } from 'chai'
|
||||||
|
|
||||||
const { expect } = require('chai')
|
export default {
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
uploadStringToPersistor,
|
uploadStringToPersistor,
|
||||||
getStringFromPersistor,
|
getStringFromPersistor,
|
||||||
expectPersistorToHaveFile,
|
expectPersistorToHaveFile,
|
||||||
|
|||||||
@@ -1,39 +1,8 @@
|
|||||||
const sinon = require('sinon')
|
import chai from 'chai'
|
||||||
const SandboxedModule = require('sandboxed-module')
|
import mongodb from 'mongodb'
|
||||||
|
import chaiAsPromised from 'chai-as-promised'
|
||||||
|
|
||||||
|
chai.use(chaiAsPromised)
|
||||||
|
|
||||||
// ensure every ObjectId has the id string as a property for correct comparisons
|
// ensure every ObjectId has the id string as a property for correct comparisons
|
||||||
require('mongodb').ObjectId.cacheHexString = true
|
mongodb.ObjectId.cacheHexString = true
|
||||||
|
|
||||||
const sandbox = sinon.createSandbox()
|
|
||||||
const stubs = {
|
|
||||||
logger: {
|
|
||||||
debug: sandbox.stub(),
|
|
||||||
log: sandbox.stub(),
|
|
||||||
info: sandbox.stub(),
|
|
||||||
warn: sandbox.stub(),
|
|
||||||
err: sandbox.stub(),
|
|
||||||
error: sandbox.stub(),
|
|
||||||
fatal: sandbox.stub(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
SandboxedModule.configure({
|
|
||||||
requires: {
|
|
||||||
'@overleaf/logger': stubs.logger,
|
|
||||||
},
|
|
||||||
sourceTransformers: {
|
|
||||||
removeNodePrefix: function (source) {
|
|
||||||
return source.replace(/require\(['"]node:/g, "require('")
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
exports.mochaHooks = {
|
|
||||||
beforeEach() {
|
|
||||||
this.logger = stubs.logger
|
|
||||||
},
|
|
||||||
|
|
||||||
afterEach() {
|
|
||||||
sandbox.reset()
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,11 +1,10 @@
|
|||||||
const sinon = require('sinon')
|
import sinon from 'sinon'
|
||||||
const chai = require('chai')
|
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
const { expect } = chai
|
import Errors from '../../../app/js/Errors.js'
|
||||||
const SandboxedModule = require('sandboxed-module')
|
|
||||||
const Errors = require('../../../app/js/Errors')
|
|
||||||
const modulePath = '../../../app/js/FileController.js'
|
const modulePath = '../../../app/js/FileController.js'
|
||||||
|
|
||||||
describe('FileController', function () {
|
describe('FileController', () => {
|
||||||
let FileHandler, LocalFileWriter, FileController, req, res, next, stream
|
let FileHandler, LocalFileWriter, FileController, req, res, next, stream
|
||||||
const settings = {
|
const settings = {
|
||||||
s3: {
|
s3: {
|
||||||
@@ -24,7 +23,7 @@ describe('FileController', function () {
|
|||||||
const key = `${projectId}/${fileId}`
|
const key = `${projectId}/${fileId}`
|
||||||
const error = new Error('incorrect utensil')
|
const error = new Error('incorrect utensil')
|
||||||
|
|
||||||
beforeEach(function () {
|
beforeEach(async () => {
|
||||||
FileHandler = {
|
FileHandler = {
|
||||||
getFile: sinon.stub().yields(null, fileStream),
|
getFile: sinon.stub().yields(null, fileStream),
|
||||||
getFileSize: sinon.stub().yields(null, fileSize),
|
getFileSize: sinon.stub().yields(null, fileSize),
|
||||||
@@ -37,19 +36,31 @@ describe('FileController', function () {
|
|||||||
pipeline: sinon.stub(),
|
pipeline: sinon.stub(),
|
||||||
}
|
}
|
||||||
|
|
||||||
FileController = SandboxedModule.require(modulePath, {
|
vi.doMock('../../../app/js/LocalFileWriter', () => ({
|
||||||
requires: {
|
default: LocalFileWriter,
|
||||||
'./LocalFileWriter': LocalFileWriter,
|
}))
|
||||||
'./FileHandler': FileHandler,
|
|
||||||
'./Errors': Errors,
|
vi.doMock('../../../app/js/FileHandler', () => ({
|
||||||
stream,
|
default: FileHandler,
|
||||||
'@overleaf/settings': settings,
|
}))
|
||||||
'@overleaf/metrics': {
|
|
||||||
inc() {},
|
vi.doMock('../../../app/js/Errors', () => ({
|
||||||
},
|
default: Errors,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('stream', () => stream)
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/settings', () => ({
|
||||||
|
default: settings,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/metrics', () => ({
|
||||||
|
default: {
|
||||||
|
inc() {},
|
||||||
},
|
},
|
||||||
globals: { console },
|
}))
|
||||||
})
|
|
||||||
|
FileController = (await import(modulePath)).default
|
||||||
|
|
||||||
req = {
|
req = {
|
||||||
key,
|
key,
|
||||||
@@ -76,76 +87,78 @@ describe('FileController', function () {
|
|||||||
next = sinon.stub()
|
next = sinon.stub()
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('getFile', function () {
|
describe('getFile', () => {
|
||||||
it('should try and get a redirect url first', function () {
|
it('should try and get a redirect url first', () => {
|
||||||
FileController.getFile(req, res, next)
|
FileController.getFile(req, res, next)
|
||||||
expect(FileHandler.getRedirectUrl).to.have.been.calledWith(bucket, key)
|
expect(FileHandler.getRedirectUrl).to.have.been.calledWith(bucket, key)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should pipe the stream', function () {
|
it('should pipe the stream', () => {
|
||||||
FileController.getFile(req, res, next)
|
FileController.getFile(req, res, next)
|
||||||
expect(stream.pipeline).to.have.been.calledWith(fileStream, res)
|
expect(stream.pipeline).to.have.been.calledWith(fileStream, res)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should send a 200 if the cacheWarm param is true', function (done) {
|
it('should send a 200 if the cacheWarm param is true', async () => {
|
||||||
req.query.cacheWarm = true
|
req.query.cacheWarm = true
|
||||||
res.sendStatus = statusCode => {
|
await new Promise(resolve => {
|
||||||
statusCode.should.equal(200)
|
res.sendStatus = statusCode => {
|
||||||
done()
|
expect(statusCode).to.equal(200)
|
||||||
}
|
resolve()
|
||||||
FileController.getFile(req, res, next)
|
}
|
||||||
|
FileController.getFile(req, res, next)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should send an error if there is a problem', function () {
|
it('should send an error if there is a problem', () => {
|
||||||
FileHandler.getFile.yields(error)
|
FileHandler.getFile.yields(error)
|
||||||
FileController.getFile(req, res, next)
|
FileController.getFile(req, res, next)
|
||||||
expect(next).to.have.been.calledWith(error)
|
expect(next).to.have.been.calledWith(error)
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('with a redirect url', function () {
|
describe('with a redirect url', () => {
|
||||||
const redirectUrl = 'https://wombat.potato/giraffe'
|
const redirectUrl = 'https://wombat.potato/giraffe'
|
||||||
|
|
||||||
beforeEach(function () {
|
beforeEach(() => {
|
||||||
FileHandler.getRedirectUrl.yields(null, redirectUrl)
|
FileHandler.getRedirectUrl.yields(null, redirectUrl)
|
||||||
res.redirect = sinon.stub()
|
res.redirect = sinon.stub()
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should redirect', function () {
|
it('should redirect', () => {
|
||||||
FileController.getFile(req, res, next)
|
FileController.getFile(req, res, next)
|
||||||
expect(res.redirect).to.have.been.calledWith(redirectUrl)
|
expect(res.redirect).to.have.been.calledWith(redirectUrl)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should not get a file stream', function () {
|
it('should not get a file stream', () => {
|
||||||
FileController.getFile(req, res, next)
|
FileController.getFile(req, res, next)
|
||||||
expect(FileHandler.getFile).not.to.have.been.called
|
expect(FileHandler.getFile).not.to.have.been.called
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('when there is an error getting the redirect url', function () {
|
describe('when there is an error getting the redirect url', () => {
|
||||||
beforeEach(function () {
|
beforeEach(() => {
|
||||||
FileHandler.getRedirectUrl.yields(new Error('wombat herding error'))
|
FileHandler.getRedirectUrl.yields(new Error('wombat herding error'))
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should not redirect', function () {
|
it('should not redirect', () => {
|
||||||
FileController.getFile(req, res, next)
|
FileController.getFile(req, res, next)
|
||||||
expect(res.redirect).not.to.have.been.called
|
expect(res.redirect).not.to.have.been.called
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should not return an error', function () {
|
it('should not return an error', () => {
|
||||||
FileController.getFile(req, res, next)
|
FileController.getFile(req, res, next)
|
||||||
expect(next).not.to.have.been.called
|
expect(next).not.to.have.been.called
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should proxy the file', function () {
|
it('should proxy the file', () => {
|
||||||
FileController.getFile(req, res, next)
|
FileController.getFile(req, res, next)
|
||||||
expect(FileHandler.getFile).to.have.been.calledWith(bucket, key)
|
expect(FileHandler.getFile).to.have.been.calledWith(bucket, key)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('with a range header', function () {
|
describe('with a range header', () => {
|
||||||
let expectedOptions
|
let expectedOptions
|
||||||
|
|
||||||
beforeEach(function () {
|
beforeEach(() => {
|
||||||
expectedOptions = {
|
expectedOptions = {
|
||||||
bucket,
|
bucket,
|
||||||
key,
|
key,
|
||||||
@@ -154,7 +167,7 @@ describe('FileController', function () {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should pass range options to FileHandler', function () {
|
it('should pass range options to FileHandler', () => {
|
||||||
req.headers.range = 'bytes=0-8'
|
req.headers.range = 'bytes=0-8'
|
||||||
expectedOptions.start = 0
|
expectedOptions.start = 0
|
||||||
expectedOptions.end = 8
|
expectedOptions.end = 8
|
||||||
@@ -167,7 +180,7 @@ describe('FileController', function () {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should ignore an invalid range header', function () {
|
it('should ignore an invalid range header', () => {
|
||||||
req.headers.range = 'potato'
|
req.headers.range = 'potato'
|
||||||
FileController.getFile(req, res, next)
|
FileController.getFile(req, res, next)
|
||||||
expect(FileHandler.getFile).to.have.been.calledWith(
|
expect(FileHandler.getFile).to.have.been.calledWith(
|
||||||
@@ -177,7 +190,7 @@ describe('FileController', function () {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it("should ignore any type other than 'bytes'", function () {
|
it("should ignore any type other than 'bytes'", () => {
|
||||||
req.headers.range = 'wombats=0-8'
|
req.headers.range = 'wombats=0-8'
|
||||||
FileController.getFile(req, res, next)
|
FileController.getFile(req, res, next)
|
||||||
expect(FileHandler.getFile).to.have.been.calledWith(
|
expect(FileHandler.getFile).to.have.been.calledWith(
|
||||||
@@ -189,31 +202,35 @@ describe('FileController', function () {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('getFileHead', function () {
|
describe('getFileHead', () => {
|
||||||
it('should return the file size in a Content-Length header', function (done) {
|
it('should return the file size in a Content-Length header', async () => {
|
||||||
res.end = () => {
|
await new Promise(resolve => {
|
||||||
expect(res.status).to.have.been.calledWith(200)
|
res.end = () => {
|
||||||
expect(res.set).to.have.been.calledWith('Content-Length', fileSize)
|
expect(res.status).to.have.been.calledWith(200)
|
||||||
done()
|
expect(res.set).to.have.been.calledWith('Content-Length', fileSize)
|
||||||
}
|
resolve()
|
||||||
|
}
|
||||||
|
|
||||||
FileController.getFileHead(req, res, next)
|
FileController.getFileHead(req, res, next)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should return a 404 is the file is not found', function (done) {
|
it('should return a 404 is the file is not found', async () => {
|
||||||
FileHandler.getFileSize.yields(
|
await new Promise(resolve => {
|
||||||
new Errors.NotFoundError({ message: 'not found', info: {} })
|
FileHandler.getFileSize.yields(
|
||||||
)
|
new Errors.NotFoundError({ message: 'not found', info: {} })
|
||||||
|
)
|
||||||
|
|
||||||
res.sendStatus = code => {
|
res.sendStatus = code => {
|
||||||
expect(code).to.equal(404)
|
expect(code).to.equal(404)
|
||||||
done()
|
resolve()
|
||||||
}
|
}
|
||||||
|
|
||||||
FileController.getFileHead(req, res, next)
|
FileController.getFileHead(req, res, next)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should send an error on internal errors', function () {
|
it('should send an error on internal errors', () => {
|
||||||
FileHandler.getFileSize.yields(error)
|
FileHandler.getFileSize.yields(error)
|
||||||
|
|
||||||
FileController.getFileHead(req, res, next)
|
FileController.getFileHead(req, res, next)
|
||||||
@@ -221,14 +238,20 @@ describe('FileController', function () {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('insertFile', function () {
|
describe('insertFile', () => {
|
||||||
it('should send bucket name key and res to FileHandler', function (done) {
|
it('should send bucket name key and res to FileHandler', async () => {
|
||||||
res.sendStatus = code => {
|
await new Promise(resolve => {
|
||||||
expect(FileHandler.insertFile).to.have.been.calledWith(bucket, key, req)
|
res.sendStatus = code => {
|
||||||
expect(code).to.equal(200)
|
expect(FileHandler.insertFile).to.have.been.calledWith(
|
||||||
done()
|
bucket,
|
||||||
}
|
key,
|
||||||
FileController.insertFile(req, res, next)
|
req
|
||||||
|
)
|
||||||
|
expect(code).to.equal(200)
|
||||||
|
resolve()
|
||||||
|
}
|
||||||
|
FileController.insertFile(req, res, next)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
@@ -1,12 +1,10 @@
|
|||||||
const sinon = require('sinon')
|
import sinon from 'sinon'
|
||||||
const chai = require('chai')
|
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
const { expect } = chai
|
import _ObjectPersistor, { Errors } from '@overleaf/object-persistor'
|
||||||
const SandboxedModule = require('sandboxed-module')
|
|
||||||
const { Errors } = require('@overleaf/object-persistor')
|
|
||||||
|
|
||||||
const modulePath = '../../../app/js/FileConverter.js'
|
const modulePath = '../../../app/js/FileConverter.js'
|
||||||
|
|
||||||
describe('FileConverter', function () {
|
describe('FileConverter', () => {
|
||||||
let SafeExec, FileConverter
|
let SafeExec, FileConverter
|
||||||
const sourcePath = '/data/wombat.eps'
|
const sourcePath = '/data/wombat.eps'
|
||||||
const destPath = '/tmp/dest.png'
|
const destPath = '/tmp/dest.png'
|
||||||
@@ -18,40 +16,53 @@ describe('FileConverter', function () {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
beforeEach(function () {
|
beforeEach(async () => {
|
||||||
SafeExec = {
|
SafeExec = {
|
||||||
promises: sinon.stub().resolves(destPath),
|
promises: sinon.stub().resolves(destPath),
|
||||||
}
|
}
|
||||||
|
|
||||||
const ObjectPersistor = { Errors }
|
const ObjectPersistor = { Errors }
|
||||||
|
|
||||||
FileConverter = SandboxedModule.require(modulePath, {
|
vi.doMock('../../../app/js/SafeExec', () => ({
|
||||||
requires: {
|
default: SafeExec,
|
||||||
'./SafeExec': SafeExec,
|
}))
|
||||||
'@overleaf/metrics': {
|
|
||||||
inc: sinon.stub(),
|
vi.doMock('@overleaf/metrics', () => ({
|
||||||
Timer: sinon.stub().returns({ done: sinon.stub() }),
|
default: {
|
||||||
},
|
inc: sinon.stub(),
|
||||||
'@overleaf/settings': Settings,
|
Timer: sinon.stub().returns({ done: sinon.stub() }),
|
||||||
'@overleaf/object-persistor': ObjectPersistor,
|
|
||||||
},
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/settings', async importOriginal => {
|
||||||
|
const originalModule = (await importOriginal()).default
|
||||||
|
return {
|
||||||
|
default: { ...originalModule, ...Settings },
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/object-persistor', () => ({
|
||||||
|
...ObjectPersistor,
|
||||||
|
default: _ObjectPersistor,
|
||||||
|
}))
|
||||||
|
|
||||||
|
FileConverter = (await import(modulePath)).default
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('convert', function () {
|
describe('convert', () => {
|
||||||
it('should convert the source to the requested format', async function () {
|
it('should convert the source to the requested format', async () => {
|
||||||
await FileConverter.promises.convert(sourcePath, format)
|
await FileConverter.promises.convert(sourcePath, format)
|
||||||
const args = SafeExec.promises.args[0][0]
|
const args = SafeExec.promises.args[0][0]
|
||||||
expect(args).to.include(`${sourcePath}[0]`)
|
expect(args).to.include(`${sourcePath}[0]`)
|
||||||
expect(args).to.include(`${sourcePath}.${format}`)
|
expect(args).to.include(`${sourcePath}.${format}`)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should return the dest path', async function () {
|
it('should return the dest path', async () => {
|
||||||
const destPath = await FileConverter.promises.convert(sourcePath, format)
|
const destPath = await FileConverter.promises.convert(sourcePath, format)
|
||||||
destPath.should.equal(`${sourcePath}.${format}`)
|
expect(destPath).to.equal(`${sourcePath}.${format}`)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should wrap the error from convert', async function () {
|
it('should wrap the error from convert', async () => {
|
||||||
SafeExec.promises.rejects(errorMessage)
|
SafeExec.promises.rejects(errorMessage)
|
||||||
try {
|
try {
|
||||||
await FileConverter.promises.convert(sourcePath, format)
|
await FileConverter.promises.convert(sourcePath, format)
|
||||||
@@ -62,7 +73,7 @@ describe('FileConverter', function () {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should not accept an non approved format', async function () {
|
it('should not accept an non approved format', async () => {
|
||||||
try {
|
try {
|
||||||
await FileConverter.promises.convert(sourcePath, 'potato')
|
await FileConverter.promises.convert(sourcePath, 'potato')
|
||||||
expect('error should have been thrown').not.to.exist
|
expect('error should have been thrown').not.to.exist
|
||||||
@@ -71,34 +82,32 @@ describe('FileConverter', function () {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should prefix the command with Settings.commands.convertCommandPrefix', async function () {
|
it('should prefix the command with Settings.commands.convertCommandPrefix', async () => {
|
||||||
Settings.commands.convertCommandPrefix = ['nice']
|
Settings.commands.convertCommandPrefix = ['nice']
|
||||||
await FileConverter.promises.convert(sourcePath, format)
|
await FileConverter.promises.convert(sourcePath, format)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should convert the file when called as a callback', function (done) {
|
it('should convert the file when called as a callback', async () => {
|
||||||
FileConverter.convert(sourcePath, format, (err, destPath) => {
|
const destPath = await FileConverter.promises.convert(sourcePath, format)
|
||||||
expect(err).not.to.exist
|
|
||||||
destPath.should.equal(`${sourcePath}.${format}`)
|
|
||||||
|
|
||||||
const args = SafeExec.promises.args[0][0]
|
expect(destPath).to.equal(`${sourcePath}.${format}`)
|
||||||
expect(args).to.include(`${sourcePath}[0]`)
|
|
||||||
expect(args).to.include(`${sourcePath}.${format}`)
|
const args = SafeExec.promises.args[0][0]
|
||||||
done()
|
expect(args).to.include(`${sourcePath}[0]`)
|
||||||
})
|
expect(args).to.include(`${sourcePath}.${format}`)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('thumbnail', function () {
|
describe('thumbnail', () => {
|
||||||
it('should call converter resize with args', async function () {
|
it('should call converter resize with args', async () => {
|
||||||
await FileConverter.promises.thumbnail(sourcePath)
|
await FileConverter.promises.thumbnail(sourcePath)
|
||||||
const args = SafeExec.promises.args[0][0]
|
const args = SafeExec.promises.args[0][0]
|
||||||
expect(args).to.include(`${sourcePath}[0]`)
|
expect(args).to.include(`${sourcePath}[0]`)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('preview', function () {
|
describe('preview', () => {
|
||||||
it('should call converter resize with args', async function () {
|
it('should call converter resize with args', async () => {
|
||||||
await FileConverter.promises.preview(sourcePath)
|
await FileConverter.promises.preview(sourcePath)
|
||||||
const args = SafeExec.promises.args[0][0]
|
const args = SafeExec.promises.args[0][0]
|
||||||
expect(args).to.include(`${sourcePath}[0]`)
|
expect(args).to.include(`${sourcePath}[0]`)
|
||||||
283
services/filestore/test/unit/js/FileHandler.test.js
Normal file
283
services/filestore/test/unit/js/FileHandler.test.js
Normal file
@@ -0,0 +1,283 @@
|
|||||||
|
import sinon from 'sinon'
|
||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
import { ObjectId } from 'mongodb'
|
||||||
|
import OriginalSettings from '@overleaf/settings'
|
||||||
|
|
||||||
|
const modulePath = '../../../app/js/FileHandler.js'
|
||||||
|
|
||||||
|
describe('FileHandler', () => {
|
||||||
|
let PersistorManager,
|
||||||
|
LocalFileWriter,
|
||||||
|
FileConverter,
|
||||||
|
KeyBuilder,
|
||||||
|
ImageOptimiser,
|
||||||
|
FileHandler,
|
||||||
|
Settings,
|
||||||
|
fs
|
||||||
|
|
||||||
|
const bucket = 'my_bucket'
|
||||||
|
const key = `${new ObjectId()}/${new ObjectId()}`
|
||||||
|
const convertedFolderKey = `${new ObjectId()}/${new ObjectId()}`
|
||||||
|
const sourceStream = 'sourceStream'
|
||||||
|
const convertedKey = 'convertedKey'
|
||||||
|
const redirectUrl = 'https://wombat.potato/giraffe'
|
||||||
|
const readStream = {
|
||||||
|
stream: 'readStream',
|
||||||
|
on: sinon.stub(),
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
PersistorManager = {
|
||||||
|
getObjectStream: sinon.stub().resolves(sourceStream),
|
||||||
|
getRedirectUrl: sinon.stub().resolves(redirectUrl),
|
||||||
|
checkIfObjectExists: sinon.stub().resolves(),
|
||||||
|
deleteObject: sinon.stub().resolves(),
|
||||||
|
deleteDirectory: sinon.stub().resolves(),
|
||||||
|
sendStream: sinon.stub().resolves(),
|
||||||
|
insertFile: sinon.stub().resolves(),
|
||||||
|
sendFile: sinon.stub().resolves(),
|
||||||
|
}
|
||||||
|
LocalFileWriter = {
|
||||||
|
// the callback style is used for detached cleanup calls
|
||||||
|
deleteFile: sinon.stub().yields(),
|
||||||
|
promises: {
|
||||||
|
writeStream: sinon.stub().resolves(),
|
||||||
|
deleteFile: sinon.stub().resolves(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
FileConverter = {
|
||||||
|
promises: {
|
||||||
|
convert: sinon.stub().resolves(),
|
||||||
|
thumbnail: sinon.stub().resolves(),
|
||||||
|
preview: sinon.stub().resolves(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
KeyBuilder = {
|
||||||
|
addCachingToKey: sinon.stub().returns(convertedKey),
|
||||||
|
getConvertedFolderKey: sinon.stub().returns(convertedFolderKey),
|
||||||
|
}
|
||||||
|
ImageOptimiser = {
|
||||||
|
promises: {
|
||||||
|
compressPng: sinon.stub().resolves(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
Settings = {
|
||||||
|
...OriginalSettings,
|
||||||
|
filestore: {
|
||||||
|
stores: {
|
||||||
|
...(OriginalSettings.filestore?.stores ?? {}),
|
||||||
|
template_files: 'template_files',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fs = {
|
||||||
|
createReadStream: sinon.stub().returns(readStream),
|
||||||
|
}
|
||||||
|
|
||||||
|
vi.doMock('../../../app/js/PersistorManager', () => ({
|
||||||
|
default: PersistorManager,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('../../../app/js/LocalFileWriter', () => ({
|
||||||
|
default: LocalFileWriter,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('../../../app/js/FileConverter', () => ({
|
||||||
|
default: FileConverter,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('../../../app/js/KeyBuilder', () => ({
|
||||||
|
default: KeyBuilder,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('../../../app/js/ImageOptimiser', () => ({
|
||||||
|
default: ImageOptimiser,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/settings', () => {
|
||||||
|
return {
|
||||||
|
default: Settings,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/metrics', () => ({
|
||||||
|
default: {
|
||||||
|
gauge: sinon.stub(),
|
||||||
|
Timer: sinon.stub().returns({ done: sinon.stub() }),
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('node:fs', () => ({
|
||||||
|
default: fs,
|
||||||
|
}))
|
||||||
|
|
||||||
|
FileHandler = (await import(modulePath)).default
|
||||||
|
FileHandler._TESTONLYSwapPersistorManager(PersistorManager)
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('insertFile', () => {
|
||||||
|
const stream = 'stream'
|
||||||
|
|
||||||
|
it('should send file to the filestore', async () => {
|
||||||
|
await FileHandler.promises.insertFile(bucket, key, stream)
|
||||||
|
expect(PersistorManager.sendStream).to.have.been.calledWith(
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
stream
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should not make a delete request for the convertedKey folder', async () => {
|
||||||
|
await FileHandler.promises.insertFile(bucket, key, stream)
|
||||||
|
expect(PersistorManager.deleteDirectory).not.to.have.been.called
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should accept templates-api key format', async () => {
|
||||||
|
KeyBuilder.getConvertedFolderKey.returns(
|
||||||
|
'5ecba29f1a294e007d0bccb4/v/0/pdf'
|
||||||
|
)
|
||||||
|
await FileHandler.promises.insertFile(bucket, key, stream)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should throw an error when the key is in the wrong format', async () => {
|
||||||
|
KeyBuilder.getConvertedFolderKey.returns('wombat')
|
||||||
|
expect(FileHandler.promises.insertFile(bucket, key, stream)).to.be
|
||||||
|
.rejected
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('getFile', () => {
|
||||||
|
it('should return the source stream no format or style are defined', async () => {
|
||||||
|
const stream = await FileHandler.promises.getFile(bucket, key, null)
|
||||||
|
expect(stream).to.equal(sourceStream)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should pass options through to PersistorManager', async () => {
|
||||||
|
const options = { start: 0, end: 8 }
|
||||||
|
await FileHandler.promises.getFile(bucket, key, options)
|
||||||
|
expect(PersistorManager.getObjectStream).to.have.been.calledWith(
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
options
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('when a format is defined', () => {
|
||||||
|
let result
|
||||||
|
|
||||||
|
describe('when the file is not cached', () => {
|
||||||
|
beforeEach(async () => {
|
||||||
|
const stream = await FileHandler.promises.getFile(bucket, key, {
|
||||||
|
format: 'png',
|
||||||
|
})
|
||||||
|
result = { stream }
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should convert the file', () => {
|
||||||
|
expect(FileConverter.promises.convert).to.have.been.called
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should compress the converted file', () => {
|
||||||
|
expect(ImageOptimiser.promises.compressPng).to.have.been.called
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return the the converted stream', () => {
|
||||||
|
expect(result.stream).to.equal(readStream)
|
||||||
|
expect(PersistorManager.getObjectStream).to.have.been.calledWith(
|
||||||
|
bucket,
|
||||||
|
key
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('when the file is cached', () => {
|
||||||
|
beforeEach(async () => {
|
||||||
|
PersistorManager.checkIfObjectExists = sinon.stub().resolves(true)
|
||||||
|
const stream = await FileHandler.promises.getFile(bucket, key, {
|
||||||
|
format: 'png',
|
||||||
|
})
|
||||||
|
result = { stream }
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should not convert the file', () => {
|
||||||
|
expect(FileConverter.promises.convert).not.to.have.been.called
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should not compress the converted file again', () => {
|
||||||
|
expect(ImageOptimiser.promises.compressPng).not.to.have.been.called
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return the cached stream', () => {
|
||||||
|
expect(result.stream).to.equal(sourceStream)
|
||||||
|
expect(PersistorManager.getObjectStream).to.have.been.calledWith(
|
||||||
|
bucket,
|
||||||
|
convertedKey
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('when a style is defined', () => {
|
||||||
|
it('generates a thumbnail when requested', async () => {
|
||||||
|
await FileHandler.promises.getFile(bucket, key, { style: 'thumbnail' })
|
||||||
|
expect(FileConverter.promises.thumbnail).to.have.been.called
|
||||||
|
expect(FileConverter.promises.preview).not.to.have.been.called
|
||||||
|
})
|
||||||
|
|
||||||
|
it('generates a preview when requested', async () => {
|
||||||
|
await FileHandler.promises.getFile(bucket, key, { style: 'preview' })
|
||||||
|
expect(FileConverter.promises.thumbnail).not.to.have.been.called
|
||||||
|
expect(FileConverter.promises.preview).to.have.been.called
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('getRedirectUrl', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
Settings.filestore = {
|
||||||
|
...OriginalSettings.filestore,
|
||||||
|
allowRedirects: true,
|
||||||
|
stores: {
|
||||||
|
...OriginalSettings.filestore.stores,
|
||||||
|
userFiles: bucket,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return a redirect url', async () => {
|
||||||
|
const url = await FileHandler.promises.getRedirectUrl(bucket, key)
|
||||||
|
expect(url).to.equal(redirectUrl)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should call the persistor to get a redirect url', async () => {
|
||||||
|
await FileHandler.promises.getRedirectUrl(bucket, key)
|
||||||
|
expect(PersistorManager.getRedirectUrl).to.have.been.calledWith(
|
||||||
|
bucket,
|
||||||
|
key
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return null if options are supplied', async () => {
|
||||||
|
const url = await FileHandler.promises.getRedirectUrl(bucket, key, {
|
||||||
|
start: 100,
|
||||||
|
end: 200,
|
||||||
|
})
|
||||||
|
expect(url).to.be.null
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return null if the bucket is not one of the defined ones', async () => {
|
||||||
|
const url = await FileHandler.promises.getRedirectUrl(
|
||||||
|
'a_different_bucket',
|
||||||
|
key
|
||||||
|
)
|
||||||
|
expect(url).to.be.null
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return null if redirects are not enabled', async () => {
|
||||||
|
Settings.filestore.allowRedirects = false
|
||||||
|
|
||||||
|
const url = await FileHandler.promises.getRedirectUrl(bucket, key)
|
||||||
|
expect(url).to.be.null
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -1,298 +0,0 @@
|
|||||||
const sinon = require('sinon')
|
|
||||||
const chai = require('chai')
|
|
||||||
const { expect } = chai
|
|
||||||
const modulePath = '../../../app/js/FileHandler.js'
|
|
||||||
const SandboxedModule = require('sandboxed-module')
|
|
||||||
const { ObjectId } = require('mongodb')
|
|
||||||
const { Errors } = require('@overleaf/object-persistor')
|
|
||||||
|
|
||||||
chai.use(require('sinon-chai'))
|
|
||||||
chai.use(require('chai-as-promised'))
|
|
||||||
|
|
||||||
describe('FileHandler', function () {
|
|
||||||
let PersistorManager,
|
|
||||||
LocalFileWriter,
|
|
||||||
FileConverter,
|
|
||||||
KeyBuilder,
|
|
||||||
ImageOptimiser,
|
|
||||||
FileHandler,
|
|
||||||
Settings,
|
|
||||||
fs
|
|
||||||
|
|
||||||
const bucket = 'my_bucket'
|
|
||||||
const key = `${new ObjectId()}/${new ObjectId()}`
|
|
||||||
const convertedFolderKey = `${new ObjectId()}/${new ObjectId()}`
|
|
||||||
const sourceStream = 'sourceStream'
|
|
||||||
const convertedKey = 'convertedKey'
|
|
||||||
const redirectUrl = 'https://wombat.potato/giraffe'
|
|
||||||
const readStream = {
|
|
||||||
stream: 'readStream',
|
|
||||||
on: sinon.stub(),
|
|
||||||
}
|
|
||||||
|
|
||||||
beforeEach(function () {
|
|
||||||
PersistorManager = {
|
|
||||||
getObjectStream: sinon.stub().resolves(sourceStream),
|
|
||||||
getRedirectUrl: sinon.stub().resolves(redirectUrl),
|
|
||||||
checkIfObjectExists: sinon.stub().resolves(),
|
|
||||||
deleteObject: sinon.stub().resolves(),
|
|
||||||
deleteDirectory: sinon.stub().resolves(),
|
|
||||||
sendStream: sinon.stub().resolves(),
|
|
||||||
insertFile: sinon.stub().resolves(),
|
|
||||||
sendFile: sinon.stub().resolves(),
|
|
||||||
}
|
|
||||||
LocalFileWriter = {
|
|
||||||
// the callback style is used for detached cleanup calls
|
|
||||||
deleteFile: sinon.stub().yields(),
|
|
||||||
promises: {
|
|
||||||
writeStream: sinon.stub().resolves(),
|
|
||||||
deleteFile: sinon.stub().resolves(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
FileConverter = {
|
|
||||||
promises: {
|
|
||||||
convert: sinon.stub().resolves(),
|
|
||||||
thumbnail: sinon.stub().resolves(),
|
|
||||||
preview: sinon.stub().resolves(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
KeyBuilder = {
|
|
||||||
addCachingToKey: sinon.stub().returns(convertedKey),
|
|
||||||
getConvertedFolderKey: sinon.stub().returns(convertedFolderKey),
|
|
||||||
}
|
|
||||||
ImageOptimiser = {
|
|
||||||
promises: {
|
|
||||||
compressPng: sinon.stub().resolves(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
Settings = {
|
|
||||||
filestore: {
|
|
||||||
stores: { template_files: 'template_files' },
|
|
||||||
},
|
|
||||||
}
|
|
||||||
fs = {
|
|
||||||
createReadStream: sinon.stub().returns(readStream),
|
|
||||||
}
|
|
||||||
|
|
||||||
const ObjectPersistor = { Errors }
|
|
||||||
|
|
||||||
FileHandler = SandboxedModule.require(modulePath, {
|
|
||||||
requires: {
|
|
||||||
'./PersistorManager': PersistorManager,
|
|
||||||
'./LocalFileWriter': LocalFileWriter,
|
|
||||||
'./FileConverter': FileConverter,
|
|
||||||
'./KeyBuilder': KeyBuilder,
|
|
||||||
'./ImageOptimiser': ImageOptimiser,
|
|
||||||
'@overleaf/settings': Settings,
|
|
||||||
'@overleaf/object-persistor': ObjectPersistor,
|
|
||||||
'@overleaf/metrics': {
|
|
||||||
gauge: sinon.stub(),
|
|
||||||
Timer: sinon.stub().returns({ done: sinon.stub() }),
|
|
||||||
},
|
|
||||||
fs,
|
|
||||||
},
|
|
||||||
globals: { console, process },
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('insertFile', function () {
|
|
||||||
const stream = 'stream'
|
|
||||||
|
|
||||||
it('should send file to the filestore', function (done) {
|
|
||||||
FileHandler.insertFile(bucket, key, stream, err => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(PersistorManager.sendStream).to.have.been.calledWith(
|
|
||||||
bucket,
|
|
||||||
key,
|
|
||||||
stream
|
|
||||||
)
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not make a delete request for the convertedKey folder', function (done) {
|
|
||||||
FileHandler.insertFile(bucket, key, stream, err => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(PersistorManager.deleteDirectory).not.to.have.been.called
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should accept templates-api key format', function (done) {
|
|
||||||
KeyBuilder.getConvertedFolderKey.returns(
|
|
||||||
'5ecba29f1a294e007d0bccb4/v/0/pdf'
|
|
||||||
)
|
|
||||||
FileHandler.insertFile(bucket, key, stream, err => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should throw an error when the key is in the wrong format', function (done) {
|
|
||||||
KeyBuilder.getConvertedFolderKey.returns('wombat')
|
|
||||||
FileHandler.insertFile(bucket, key, stream, err => {
|
|
||||||
expect(err).to.exist
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('getFile', function () {
|
|
||||||
it('should return the source stream no format or style are defined', function (done) {
|
|
||||||
FileHandler.getFile(bucket, key, null, (err, stream) => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(stream).to.equal(sourceStream)
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should pass options through to PersistorManager', function (done) {
|
|
||||||
const options = { start: 0, end: 8 }
|
|
||||||
FileHandler.getFile(bucket, key, options, err => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(PersistorManager.getObjectStream).to.have.been.calledWith(
|
|
||||||
bucket,
|
|
||||||
key,
|
|
||||||
options
|
|
||||||
)
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('when a format is defined', function () {
|
|
||||||
let result
|
|
||||||
|
|
||||||
describe('when the file is not cached', function () {
|
|
||||||
beforeEach(function (done) {
|
|
||||||
FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => {
|
|
||||||
result = { err, stream }
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should convert the file', function () {
|
|
||||||
expect(FileConverter.promises.convert).to.have.been.called
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should compress the converted file', function () {
|
|
||||||
expect(ImageOptimiser.promises.compressPng).to.have.been.called
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return the the converted stream', function () {
|
|
||||||
expect(result.err).not.to.exist
|
|
||||||
expect(result.stream).to.equal(readStream)
|
|
||||||
expect(PersistorManager.getObjectStream).to.have.been.calledWith(
|
|
||||||
bucket,
|
|
||||||
key
|
|
||||||
)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('when the file is cached', function () {
|
|
||||||
beforeEach(function (done) {
|
|
||||||
PersistorManager.checkIfObjectExists = sinon.stub().resolves(true)
|
|
||||||
FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => {
|
|
||||||
result = { err, stream }
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not convert the file', function () {
|
|
||||||
expect(FileConverter.promises.convert).not.to.have.been.called
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not compress the converted file again', function () {
|
|
||||||
expect(ImageOptimiser.promises.compressPng).not.to.have.been.called
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return the cached stream', function () {
|
|
||||||
expect(result.err).not.to.exist
|
|
||||||
expect(result.stream).to.equal(sourceStream)
|
|
||||||
expect(PersistorManager.getObjectStream).to.have.been.calledWith(
|
|
||||||
bucket,
|
|
||||||
convertedKey
|
|
||||||
)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('when a style is defined', function () {
|
|
||||||
it('generates a thumbnail when requested', function (done) {
|
|
||||||
FileHandler.getFile(bucket, key, { style: 'thumbnail' }, err => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(FileConverter.promises.thumbnail).to.have.been.called
|
|
||||||
expect(FileConverter.promises.preview).not.to.have.been.called
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('generates a preview when requested', function (done) {
|
|
||||||
FileHandler.getFile(bucket, key, { style: 'preview' }, err => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(FileConverter.promises.thumbnail).not.to.have.been.called
|
|
||||||
expect(FileConverter.promises.preview).to.have.been.called
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('getRedirectUrl', function () {
|
|
||||||
beforeEach(function () {
|
|
||||||
Settings.filestore = {
|
|
||||||
allowRedirects: true,
|
|
||||||
stores: {
|
|
||||||
userFiles: bucket,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return a redirect url', function (done) {
|
|
||||||
FileHandler.getRedirectUrl(bucket, key, (err, url) => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(url).to.equal(redirectUrl)
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should call the persistor to get a redirect url', function (done) {
|
|
||||||
FileHandler.getRedirectUrl(bucket, key, () => {
|
|
||||||
expect(PersistorManager.getRedirectUrl).to.have.been.calledWith(
|
|
||||||
bucket,
|
|
||||||
key
|
|
||||||
)
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return null if options are supplied', function (done) {
|
|
||||||
FileHandler.getRedirectUrl(
|
|
||||||
bucket,
|
|
||||||
key,
|
|
||||||
{ start: 100, end: 200 },
|
|
||||||
(err, url) => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(url).to.be.null
|
|
||||||
done()
|
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return null if the bucket is not one of the defined ones', function (done) {
|
|
||||||
FileHandler.getRedirectUrl('a_different_bucket', key, (err, url) => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(url).to.be.null
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return null if redirects are not enabled', function (done) {
|
|
||||||
Settings.filestore.allowRedirects = false
|
|
||||||
FileHandler.getRedirectUrl(bucket, key, (err, url) => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(url).to.be.null
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
78
services/filestore/test/unit/js/ImageOptimiser.test.js
Normal file
78
services/filestore/test/unit/js/ImageOptimiser.test.js
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
import sinon from 'sinon'
|
||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
import Errors from '../../../app/js/Errors.js'
|
||||||
|
|
||||||
|
const { FailedCommandError } = Errors
|
||||||
|
|
||||||
|
const modulePath = '../../../app/js/ImageOptimiser.js'
|
||||||
|
|
||||||
|
describe('ImageOptimiser', function () {
|
||||||
|
let ImageOptimiser, SafeExec
|
||||||
|
const sourcePath = '/wombat/potato.eps'
|
||||||
|
|
||||||
|
beforeEach(async function () {
|
||||||
|
SafeExec = {
|
||||||
|
promises: sinon.stub().resolves(),
|
||||||
|
}
|
||||||
|
|
||||||
|
vi.doMock('../../../app/js/SafeExec', () => ({
|
||||||
|
default: SafeExec,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/metrics', () => ({
|
||||||
|
default: {
|
||||||
|
Timer: sinon.stub().returns({ done: sinon.stub() }),
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
|
ImageOptimiser = (await import(modulePath)).default
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('compressPng', function () {
|
||||||
|
it('should convert the file', async function () {
|
||||||
|
await new Promise(resolve => {
|
||||||
|
ImageOptimiser.compressPng(sourcePath, err => {
|
||||||
|
expect(err).not.to.exist
|
||||||
|
expect(SafeExec.promises).to.have.been.calledWith([
|
||||||
|
'optipng',
|
||||||
|
sourcePath,
|
||||||
|
])
|
||||||
|
resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return the error', async function () {
|
||||||
|
await new Promise(resolve => {
|
||||||
|
SafeExec.promises.rejects('wombat herding failure')
|
||||||
|
ImageOptimiser.compressPng(sourcePath, err => {
|
||||||
|
expect(err.toString()).to.equal('wombat herding failure')
|
||||||
|
resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('when optimiser is sigkilled', function () {
|
||||||
|
const expectedError = new FailedCommandError('', 'SIGKILL', '', '')
|
||||||
|
let error
|
||||||
|
|
||||||
|
beforeEach(async function () {
|
||||||
|
await new Promise(resolve => {
|
||||||
|
SafeExec.promises.rejects(expectedError)
|
||||||
|
ImageOptimiser.compressPng(sourcePath, err => {
|
||||||
|
error = err
|
||||||
|
resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should not produce an error', function () {
|
||||||
|
expect(error).not.to.exist
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should log a warning', function (ctx) {
|
||||||
|
expect(ctx.logger.warn).to.have.been.calledOnce
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
const sinon = require('sinon')
|
|
||||||
const chai = require('chai')
|
|
||||||
const { expect } = chai
|
|
||||||
const modulePath = '../../../app/js/ImageOptimiser.js'
|
|
||||||
const { FailedCommandError } = require('../../../app/js/Errors')
|
|
||||||
const SandboxedModule = require('sandboxed-module')
|
|
||||||
|
|
||||||
describe('ImageOptimiser', function () {
|
|
||||||
let ImageOptimiser, SafeExec
|
|
||||||
const sourcePath = '/wombat/potato.eps'
|
|
||||||
|
|
||||||
beforeEach(function () {
|
|
||||||
SafeExec = {
|
|
||||||
promises: sinon.stub().resolves(),
|
|
||||||
}
|
|
||||||
ImageOptimiser = SandboxedModule.require(modulePath, {
|
|
||||||
requires: {
|
|
||||||
'./SafeExec': SafeExec,
|
|
||||||
'@overleaf/metrics': {
|
|
||||||
Timer: sinon.stub().returns({ done: sinon.stub() }),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('compressPng', function () {
|
|
||||||
it('should convert the file', function (done) {
|
|
||||||
ImageOptimiser.compressPng(sourcePath, err => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(SafeExec.promises).to.have.been.calledWith([
|
|
||||||
'optipng',
|
|
||||||
sourcePath,
|
|
||||||
])
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return the error', function (done) {
|
|
||||||
SafeExec.promises.rejects('wombat herding failure')
|
|
||||||
ImageOptimiser.compressPng(sourcePath, err => {
|
|
||||||
expect(err.toString()).to.equal('wombat herding failure')
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('when optimiser is sigkilled', function () {
|
|
||||||
const expectedError = new FailedCommandError('', 'SIGKILL', '', '')
|
|
||||||
let error
|
|
||||||
|
|
||||||
beforeEach(function (done) {
|
|
||||||
SafeExec.promises.rejects(expectedError)
|
|
||||||
ImageOptimiser.compressPng(sourcePath, err => {
|
|
||||||
error = err
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not produce an error', function () {
|
|
||||||
expect(error).not.to.exist
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should log a warning', function () {
|
|
||||||
expect(this.logger.warn).to.have.been.calledOnce
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
const SandboxedModule = require('sandboxed-module')
|
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
|
||||||
const modulePath = '../../../app/js/KeyBuilder.js'
|
const modulePath = '../../../app/js/KeyBuilder.js'
|
||||||
|
|
||||||
@@ -6,23 +6,25 @@ describe('KeybuilderTests', function () {
|
|||||||
let KeyBuilder
|
let KeyBuilder
|
||||||
const key = 'wombat/potato'
|
const key = 'wombat/potato'
|
||||||
|
|
||||||
beforeEach(function () {
|
beforeEach(async function () {
|
||||||
KeyBuilder = SandboxedModule.require(modulePath, {
|
vi.doMock('@overleaf/settings', () => ({
|
||||||
requires: { '@overleaf/settings': {} },
|
default: {},
|
||||||
})
|
}))
|
||||||
|
|
||||||
|
KeyBuilder = (await import(modulePath)).default
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('cachedKey', function () {
|
describe('cachedKey', function () {
|
||||||
it('should add the format to the key', function () {
|
it('should add the format to the key', function () {
|
||||||
const opts = { format: 'png' }
|
const opts = { format: 'png' }
|
||||||
const newKey = KeyBuilder.addCachingToKey(key, opts)
|
const newKey = KeyBuilder.addCachingToKey(key, opts)
|
||||||
newKey.should.equal(`${key}-converted-cache/format-png`)
|
expect(newKey).to.equal(`${key}-converted-cache/format-png`)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should add the style to the key', function () {
|
it('should add the style to the key', function () {
|
||||||
const opts = { style: 'thumbnail' }
|
const opts = { style: 'thumbnail' }
|
||||||
const newKey = KeyBuilder.addCachingToKey(key, opts)
|
const newKey = KeyBuilder.addCachingToKey(key, opts)
|
||||||
newKey.should.equal(`${key}-converted-cache/style-thumbnail`)
|
expect(newKey).to.equal(`${key}-converted-cache/style-thumbnail`)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should add format first, then style', function () {
|
it('should add format first, then style', function () {
|
||||||
@@ -31,7 +33,9 @@ describe('KeybuilderTests', function () {
|
|||||||
format: 'png',
|
format: 'png',
|
||||||
}
|
}
|
||||||
const newKey = KeyBuilder.addCachingToKey(key, opts)
|
const newKey = KeyBuilder.addCachingToKey(key, opts)
|
||||||
newKey.should.equal(`${key}-converted-cache/format-png-style-thumbnail`)
|
expect(newKey).to.equal(
|
||||||
|
`${key}-converted-cache/format-png-style-thumbnail`
|
||||||
|
)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
115
services/filestore/test/unit/js/LocalFileWriter.test.js
Normal file
115
services/filestore/test/unit/js/LocalFileWriter.test.js
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
import sinon from 'sinon'
|
||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
import { Errors } from '@overleaf/object-persistor'
|
||||||
|
|
||||||
|
const modulePath = '../../../app/js/LocalFileWriter.js'
|
||||||
|
|
||||||
|
describe('LocalFileWriter', function () {
|
||||||
|
const writeStream = 'writeStream'
|
||||||
|
const readStream = 'readStream'
|
||||||
|
const settings = { path: { uploadFolder: '/uploads' } }
|
||||||
|
const fsPath = '/uploads/wombat'
|
||||||
|
const filename = 'wombat'
|
||||||
|
let stream, fs, LocalFileWriter
|
||||||
|
|
||||||
|
beforeEach(async function () {
|
||||||
|
fs = {
|
||||||
|
createWriteStream: sinon.stub().returns(writeStream),
|
||||||
|
promises: {
|
||||||
|
unlink: sinon.stub().resolves(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
stream = {
|
||||||
|
pipeline: sinon.stub().yields(),
|
||||||
|
}
|
||||||
|
|
||||||
|
const ObjectPersistor = { Errors }
|
||||||
|
|
||||||
|
vi.doMock('fs', () => ({
|
||||||
|
default: fs,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('stream', () => ({
|
||||||
|
default: stream,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/settings', () => ({
|
||||||
|
default: settings,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/metrics', () => ({
|
||||||
|
default: {
|
||||||
|
inc: sinon.stub(),
|
||||||
|
Timer: sinon.stub().returns({ done: sinon.stub() }),
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/object-persistor', () => ObjectPersistor)
|
||||||
|
|
||||||
|
LocalFileWriter = (await import(modulePath)).default
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('writeStream', function () {
|
||||||
|
it('writes the stream to the upload folder', async function () {
|
||||||
|
await new Promise(resolve => {
|
||||||
|
LocalFileWriter.writeStream(readStream, filename, (err, path) => {
|
||||||
|
expect(err).not.to.exist
|
||||||
|
expect(fs.createWriteStream).to.have.been.calledWith(fsPath)
|
||||||
|
expect(stream.pipeline).to.have.been.calledWith(
|
||||||
|
readStream,
|
||||||
|
writeStream
|
||||||
|
)
|
||||||
|
expect(path).to.equal(fsPath)
|
||||||
|
resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('when there is an error', function () {
|
||||||
|
const error = new Error('not enough ketchup')
|
||||||
|
beforeEach(function () {
|
||||||
|
stream.pipeline.yields(error)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should wrap the error', async function () {
|
||||||
|
await expect(
|
||||||
|
LocalFileWriter.promises.writeStream(readStream, filename)
|
||||||
|
).to.be.rejected.and.eventually.have.property('cause', error)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should delete the temporary file', async function () {
|
||||||
|
await expect(LocalFileWriter.promises.writeStream(readStream, filename))
|
||||||
|
.to.be.rejected
|
||||||
|
expect(fs.promises.unlink).to.have.been.calledWith(fsPath)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('deleteFile', function () {
|
||||||
|
it('should unlink the file', async function () {
|
||||||
|
await LocalFileWriter.promises.deleteFile(fsPath)
|
||||||
|
expect(fs.promises.unlink).to.have.been.calledWith(fsPath)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should not call unlink with an empty path', async function () {
|
||||||
|
await LocalFileWriter.promises.deleteFile('')
|
||||||
|
|
||||||
|
expect(fs.promises.unlink).not.to.have.been.called
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should not throw a error if the file does not exist', async function () {
|
||||||
|
const error = new Error('file not found')
|
||||||
|
error.code = 'ENOENT'
|
||||||
|
fs.promises.unlink = sinon.stub().rejects(error)
|
||||||
|
await LocalFileWriter.promises.deleteFile(fsPath)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should wrap the error', async function () {
|
||||||
|
const error = new Error('failed to reticulate splines')
|
||||||
|
fs.promises.unlink = sinon.stub().rejects(error)
|
||||||
|
await expect(
|
||||||
|
LocalFileWriter.promises.deleteFile(fsPath)
|
||||||
|
).to.be.rejectedWith(Errors.WriteError)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
const sinon = require('sinon')
|
|
||||||
const chai = require('chai')
|
|
||||||
const { expect } = chai
|
|
||||||
const modulePath = '../../../app/js/LocalFileWriter.js'
|
|
||||||
const SandboxedModule = require('sandboxed-module')
|
|
||||||
const { Errors } = require('@overleaf/object-persistor')
|
|
||||||
chai.use(require('sinon-chai'))
|
|
||||||
|
|
||||||
describe('LocalFileWriter', function () {
|
|
||||||
const writeStream = 'writeStream'
|
|
||||||
const readStream = 'readStream'
|
|
||||||
const settings = { path: { uploadFolder: '/uploads' } }
|
|
||||||
const fsPath = '/uploads/wombat'
|
|
||||||
const filename = 'wombat'
|
|
||||||
let stream, fs, LocalFileWriter
|
|
||||||
|
|
||||||
beforeEach(function () {
|
|
||||||
fs = {
|
|
||||||
createWriteStream: sinon.stub().returns(writeStream),
|
|
||||||
unlink: sinon.stub().yields(),
|
|
||||||
}
|
|
||||||
stream = {
|
|
||||||
pipeline: sinon.stub().yields(),
|
|
||||||
}
|
|
||||||
|
|
||||||
const ObjectPersistor = { Errors }
|
|
||||||
|
|
||||||
LocalFileWriter = SandboxedModule.require(modulePath, {
|
|
||||||
requires: {
|
|
||||||
fs,
|
|
||||||
stream,
|
|
||||||
'@overleaf/settings': settings,
|
|
||||||
'@overleaf/metrics': {
|
|
||||||
inc: sinon.stub(),
|
|
||||||
Timer: sinon.stub().returns({ done: sinon.stub() }),
|
|
||||||
},
|
|
||||||
'@overleaf/object-persistor': ObjectPersistor,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('writeStream', function () {
|
|
||||||
it('writes the stream to the upload folder', function (done) {
|
|
||||||
LocalFileWriter.writeStream(readStream, filename, (err, path) => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(fs.createWriteStream).to.have.been.calledWith(fsPath)
|
|
||||||
expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream)
|
|
||||||
expect(path).to.equal(fsPath)
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('when there is an error', function () {
|
|
||||||
const error = new Error('not enough ketchup')
|
|
||||||
beforeEach(function () {
|
|
||||||
stream.pipeline.yields(error)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should wrap the error', function () {
|
|
||||||
LocalFileWriter.writeStream(readStream, filename, err => {
|
|
||||||
expect(err).to.exist
|
|
||||||
expect(err.cause).to.equal(error)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should delete the temporary file', function () {
|
|
||||||
LocalFileWriter.writeStream(readStream, filename, () => {
|
|
||||||
expect(fs.unlink).to.have.been.calledWith(fsPath)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('deleteFile', function () {
|
|
||||||
it('should unlink the file', function (done) {
|
|
||||||
LocalFileWriter.deleteFile(fsPath, err => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(fs.unlink).to.have.been.calledWith(fsPath)
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not call unlink with an empty path', function (done) {
|
|
||||||
LocalFileWriter.deleteFile('', err => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
expect(fs.unlink).not.to.have.been.called
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not throw a error if the file does not exist', function (done) {
|
|
||||||
const error = new Error('file not found')
|
|
||||||
error.code = 'ENOENT'
|
|
||||||
fs.unlink = sinon.stub().yields(error)
|
|
||||||
LocalFileWriter.deleteFile(fsPath, err => {
|
|
||||||
expect(err).not.to.exist
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should wrap the error', function (done) {
|
|
||||||
const error = new Error('failed to reticulate splines')
|
|
||||||
fs.unlink = sinon.stub().yields(error)
|
|
||||||
LocalFileWriter.deleteFile(fsPath, err => {
|
|
||||||
expect(err).to.exist
|
|
||||||
expect(err.cause).to.equal(error)
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
114
services/filestore/test/unit/js/SafeExec.test.js
Normal file
114
services/filestore/test/unit/js/SafeExec.test.js
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
import { beforeEach, chai, describe, expect, it, vi } from 'vitest'
|
||||||
|
|
||||||
|
const should = chai.should()
|
||||||
|
const modulePath = '../../../app/js/SafeExec.js'
|
||||||
|
|
||||||
|
describe('SafeExec', function () {
|
||||||
|
let settings, options, safeExec
|
||||||
|
|
||||||
|
beforeEach(async function () {
|
||||||
|
settings = { enableConversions: true }
|
||||||
|
options = { timeout: 10 * 1000, killSignal: 'SIGTERM' }
|
||||||
|
|
||||||
|
vi.doMock('@overleaf/settings', () => ({
|
||||||
|
default: settings,
|
||||||
|
}))
|
||||||
|
|
||||||
|
safeExec = (await import(modulePath)).default
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('safeExec', function () {
|
||||||
|
it('should execute a valid command', async function () {
|
||||||
|
await new Promise(resolve => {
|
||||||
|
safeExec(['/bin/echo', 'hello'], options, (err, stdout, stderr) => {
|
||||||
|
stdout.should.equal('hello\n')
|
||||||
|
stderr.should.equal('')
|
||||||
|
should.not.exist(err)
|
||||||
|
resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should error when conversions are disabled', async function () {
|
||||||
|
await new Promise(resolve => {
|
||||||
|
settings.enableConversions = false
|
||||||
|
safeExec(['/bin/echo', 'hello'], options, err => {
|
||||||
|
expect(err).to.exist
|
||||||
|
resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should execute a command with non-zero exit status', async function () {
|
||||||
|
await new Promise(resolve => {
|
||||||
|
safeExec(['/usr/bin/env', 'false'], options, err => {
|
||||||
|
expect(err).to.exist
|
||||||
|
expect(err.name).to.equal('FailedCommandError')
|
||||||
|
expect(err.code).to.equal(1)
|
||||||
|
expect(err.stdout).to.equal('')
|
||||||
|
expect(err.stderr).to.equal('')
|
||||||
|
resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should handle an invalid command', async function () {
|
||||||
|
await new Promise(resolve => {
|
||||||
|
safeExec(['/bin/foobar'], options, err => {
|
||||||
|
err.code.should.equal('ENOENT')
|
||||||
|
resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should handle a command that runs too long', async function () {
|
||||||
|
await new Promise(resolve => {
|
||||||
|
safeExec(
|
||||||
|
['/bin/sleep', '10'],
|
||||||
|
{ timeout: 500, killSignal: 'SIGTERM' },
|
||||||
|
err => {
|
||||||
|
expect(err).to.exist
|
||||||
|
expect(err.name).to.equal('FailedCommandError')
|
||||||
|
expect(err.code).to.equal('SIGTERM')
|
||||||
|
resolve()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('as a promise', function () {
|
||||||
|
beforeEach(function () {
|
||||||
|
safeExec = safeExec.promises
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should execute a valid command', async function () {
|
||||||
|
const { stdout, stderr } = await safeExec(['/bin/echo', 'hello'], options)
|
||||||
|
|
||||||
|
stdout.should.equal('hello\n')
|
||||||
|
stderr.should.equal('')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should throw a ConversionsDisabledError when appropriate', async function () {
|
||||||
|
settings.enableConversions = false
|
||||||
|
try {
|
||||||
|
await safeExec(['/bin/echo', 'hello'], options)
|
||||||
|
} catch (err) {
|
||||||
|
expect(err.name).to.equal('ConversionsDisabledError')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
expect('method did not throw an error').not.to.exist
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should throw a FailedCommandError when appropriate', async function () {
|
||||||
|
try {
|
||||||
|
await safeExec(['/usr/bin/env', 'false'], options)
|
||||||
|
} catch (err) {
|
||||||
|
expect(err.name).to.equal('FailedCommandError')
|
||||||
|
expect(err.code).to.equal(1)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
expect('method did not throw an error').not.to.exist
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -1,110 +0,0 @@
|
|||||||
const chai = require('chai')
|
|
||||||
const should = chai.should()
|
|
||||||
const { expect } = chai
|
|
||||||
const modulePath = '../../../app/js/SafeExec'
|
|
||||||
const { Errors } = require('@overleaf/object-persistor')
|
|
||||||
const SandboxedModule = require('sandboxed-module')
|
|
||||||
|
|
||||||
describe('SafeExec', function () {
|
|
||||||
let settings, options, safeExec
|
|
||||||
|
|
||||||
beforeEach(function () {
|
|
||||||
settings = { enableConversions: true }
|
|
||||||
options = { timeout: 10 * 1000, killSignal: 'SIGTERM' }
|
|
||||||
|
|
||||||
const ObjectPersistor = { Errors }
|
|
||||||
|
|
||||||
safeExec = SandboxedModule.require(modulePath, {
|
|
||||||
globals: { process },
|
|
||||||
requires: {
|
|
||||||
'@overleaf/settings': settings,
|
|
||||||
'@overleaf/object-persistor': ObjectPersistor,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('safeExec', function () {
|
|
||||||
it('should execute a valid command', function (done) {
|
|
||||||
safeExec(['/bin/echo', 'hello'], options, (err, stdout, stderr) => {
|
|
||||||
stdout.should.equal('hello\n')
|
|
||||||
stderr.should.equal('')
|
|
||||||
should.not.exist(err)
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should error when conversions are disabled', function (done) {
|
|
||||||
settings.enableConversions = false
|
|
||||||
safeExec(['/bin/echo', 'hello'], options, err => {
|
|
||||||
expect(err).to.exist
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should execute a command with non-zero exit status', function (done) {
|
|
||||||
safeExec(['/usr/bin/env', 'false'], options, err => {
|
|
||||||
expect(err).to.exist
|
|
||||||
expect(err.name).to.equal('FailedCommandError')
|
|
||||||
expect(err.code).to.equal(1)
|
|
||||||
expect(err.stdout).to.equal('')
|
|
||||||
expect(err.stderr).to.equal('')
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle an invalid command', function (done) {
|
|
||||||
safeExec(['/bin/foobar'], options, err => {
|
|
||||||
err.code.should.equal('ENOENT')
|
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle a command that runs too long', function (done) {
|
|
||||||
safeExec(
|
|
||||||
['/bin/sleep', '10'],
|
|
||||||
{ timeout: 500, killSignal: 'SIGTERM' },
|
|
||||||
err => {
|
|
||||||
expect(err).to.exist
|
|
||||||
expect(err.name).to.equal('FailedCommandError')
|
|
||||||
expect(err.code).to.equal('SIGTERM')
|
|
||||||
done()
|
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('as a promise', function () {
|
|
||||||
beforeEach(function () {
|
|
||||||
safeExec = safeExec.promises
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should execute a valid command', async function () {
|
|
||||||
const { stdout, stderr } = await safeExec(['/bin/echo', 'hello'], options)
|
|
||||||
|
|
||||||
stdout.should.equal('hello\n')
|
|
||||||
stderr.should.equal('')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should throw a ConversionsDisabledError when appropriate', async function () {
|
|
||||||
settings.enableConversions = false
|
|
||||||
try {
|
|
||||||
await safeExec(['/bin/echo', 'hello'], options)
|
|
||||||
} catch (err) {
|
|
||||||
expect(err.name).to.equal('ConversionsDisabledError')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
expect('method did not throw an error').not.to.exist
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should throw a FailedCommandError when appropriate', async function () {
|
|
||||||
try {
|
|
||||||
await safeExec(['/usr/bin/env', 'false'], options)
|
|
||||||
} catch (err) {
|
|
||||||
expect(err.name).to.equal('FailedCommandError')
|
|
||||||
expect(err.code).to.equal(1)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
expect('method did not throw an error').not.to.exist
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
24
services/filestore/test/unit/js/Settings.test.js
Normal file
24
services/filestore/test/unit/js/Settings.test.js
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
|
||||||
|
describe('Settings', function () {
|
||||||
|
describe('s3', function () {
|
||||||
|
const s3Settings = {
|
||||||
|
bucket1: {
|
||||||
|
auth_key: 'bucket1_key',
|
||||||
|
auth_secret: 'bucket1_secret',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllEnvs()
|
||||||
|
})
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.stubEnv()
|
||||||
|
process.env.S3_BUCKET_CREDENTIALS = JSON.stringify(s3Settings)
|
||||||
|
})
|
||||||
|
it('should use JSONified env var if present', async function () {
|
||||||
|
const settings = (await import('@overleaf/settings')).default
|
||||||
|
expect(settings.filestore.s3.bucketCreds).to.deep.equal(s3Settings)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
const chai = require('chai')
|
|
||||||
const { expect } = chai
|
|
||||||
const SandboxedModule = require('sandboxed-module')
|
|
||||||
|
|
||||||
describe('Settings', function () {
|
|
||||||
describe('s3', function () {
|
|
||||||
it('should use JSONified env var if present', function () {
|
|
||||||
const s3Settings = {
|
|
||||||
bucket1: {
|
|
||||||
auth_key: 'bucket1_key',
|
|
||||||
auth_secret: 'bucket1_secret',
|
|
||||||
},
|
|
||||||
}
|
|
||||||
process.env.S3_BUCKET_CREDENTIALS = JSON.stringify(s3Settings)
|
|
||||||
const settings = SandboxedModule.require('@overleaf/settings', {
|
|
||||||
globals: { console, process },
|
|
||||||
})
|
|
||||||
expect(settings.filestore.s3.bucketCreds).to.deep.equal(s3Settings)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
37
services/filestore/test/unit/setup.js
Normal file
37
services/filestore/test/unit/setup.js
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
import { beforeEach, afterEach, chai, vi } from 'vitest'
|
||||||
|
import sinon from 'sinon'
|
||||||
|
import mongodb from 'mongodb'
|
||||||
|
import sinonChai from 'sinon-chai'
|
||||||
|
import chaiAsPromised from 'chai-as-promised'
|
||||||
|
|
||||||
|
chai.use(sinonChai)
|
||||||
|
chai.use(chaiAsPromised)
|
||||||
|
|
||||||
|
// ensure every ObjectId has the id string as a property for correct comparisons
|
||||||
|
mongodb.ObjectId.cacheHexString = true
|
||||||
|
|
||||||
|
const sandbox = sinon.createSandbox()
|
||||||
|
const stubs = {
|
||||||
|
logger: {
|
||||||
|
debug: sandbox.stub(),
|
||||||
|
log: sandbox.stub(),
|
||||||
|
info: sandbox.stub(),
|
||||||
|
warn: sandbox.stub(),
|
||||||
|
err: sandbox.stub(),
|
||||||
|
error: sandbox.stub(),
|
||||||
|
fatal: sandbox.stub(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(ctx => {
|
||||||
|
ctx.logger = stubs.logger
|
||||||
|
vi.doMock('@overleaf/logger', () => ({
|
||||||
|
default: ctx.logger,
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
sandbox.reset()
|
||||||
|
vi.restoreAllMocks()
|
||||||
|
vi.resetModules()
|
||||||
|
})
|
||||||
@@ -8,6 +8,8 @@
|
|||||||
"config/**/*",
|
"config/**/*",
|
||||||
"scripts/**/*",
|
"scripts/**/*",
|
||||||
"test/**/*",
|
"test/**/*",
|
||||||
"types"
|
"types",
|
||||||
|
"vitest.config.acceptance.cjs",
|
||||||
|
"vitest.config.unit.cjs"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
25
services/filestore/vitest.config.unit.cjs
Normal file
25
services/filestore/vitest.config.unit.cjs
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
const { defineConfig } = require('vitest/config')
|
||||||
|
|
||||||
|
let reporterOptions = {}
|
||||||
|
if (process.env.CI) {
|
||||||
|
reporterOptions = {
|
||||||
|
reporters: [
|
||||||
|
'default',
|
||||||
|
[
|
||||||
|
'junit',
|
||||||
|
{
|
||||||
|
classnameTemplate: `Unit tests.{filename}`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
outputFile: 'reports/junit-vitest-unit.xml',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
module.exports = defineConfig({
|
||||||
|
test: {
|
||||||
|
include: ['test/unit/js/*.test.{js,ts}'],
|
||||||
|
setupFiles: ['./test/unit/setup.js'],
|
||||||
|
isolate: true,
|
||||||
|
...reporterOptions,
|
||||||
|
},
|
||||||
|
})
|
||||||
Reference in New Issue
Block a user