diff --git a/package-lock.json b/package-lock.json index 9933b92406..3a6fa61776 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34693,6 +34693,7 @@ "express": "^4.18.2", "lodash": "^4.17.21", "lru-cache": "^4.1.5", + "minimatch": "^7.4.2", "mongodb": "^4.11.0", "node-fetch": "^2.6.7", "p-limit": "^2.3.0", @@ -34709,6 +34710,14 @@ "sinon-stub-promise": "^4.0.0" } }, + "services/third-party-datastore/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "services/third-party-datastore/node_modules/bson": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz", @@ -34769,6 +34778,20 @@ "yallist": "^2.1.2" } }, + "services/third-party-datastore/node_modules/minimatch": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz", + "integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "services/third-party-datastore/node_modules/mongodb": { "version": "4.11.0", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz", @@ -35210,6 +35233,7 @@ "mathjax": "^2.7.9", "mathjax-3": "npm:mathjax@^3.2.2", "method-override": "^2.3.3", + "minimatch": "^7.4.2", "minimist": "^1.2.7", "mmmagic": "^0.5.3", "moment": "^2.29.4", @@ -36405,6 +36429,18 @@ "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8" } }, + "services/web/node_modules/eslint-plugin-jsx-a11y/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "services/web/node_modules/eslint-plugin-react": { "version": "7.28.0", "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.28.0.tgz", @@ -36457,6 +36493,18 @@ "node": ">=0.10.0" } }, + "services/web/node_modules/eslint-plugin-react/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "services/web/node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -36690,6 +36738,18 @@ "webpack": "^5.0.0" } }, + "services/web/node_modules/karma-webpack/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "services/web/node_modules/karma-webpack/node_modules/webpack-merge": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.2.2.tgz", @@ -36832,6 +36892,28 @@ "url": "https://opencollective.com/webpack" } }, + "services/web/node_modules/minimatch": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz", + "integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "services/web/node_modules/minimatch/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "services/web/node_modules/mongodb": { "version": "4.13.0", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz", @@ -44467,6 +44549,7 @@ "express": "^4.18.2", "lodash": "^4.17.21", "lru-cache": "^4.1.5", + "minimatch": "*", "mocha": "^10.2.0", "mongodb": "^4.11.0", "nock": "0.15.2", @@ -44479,6 +44562,14 @@ "sinon-stub-promise": "^4.0.0" }, "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, "bson": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz", @@ -44516,6 +44607,14 @@ "yallist": "^2.1.2" } }, + "minimatch": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz", + "integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==", + "requires": { + "brace-expansion": "^2.0.1" + } + }, "mongodb": { "version": "4.11.0", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz", @@ -44958,6 +45057,7 @@ "mensch": "^0.3.4", "method-override": "^2.3.3", "mini-css-extract-plugin": "^2.6.0", + "minimatch": "^7.4.2", "minimist": "^1.2.7", "mmmagic": "^0.5.3", "mocha": "^10.2.0", @@ -45772,6 +45872,17 @@ "jsx-ast-utils": "^3.2.1", "language-tags": "^1.0.5", "minimatch": "^3.0.4" + }, + "dependencies": { + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + } } }, "eslint-plugin-react": { @@ -45804,6 +45915,15 @@ "requires": { "esutils": "^2.0.2" } + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } } } }, @@ -45976,6 +46096,15 @@ "webpack-merge": "^4.1.5" }, "dependencies": { + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, "webpack-merge": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.2.2.tgz", @@ -46083,6 +46212,24 @@ } } }, + "minimatch": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz", + "integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==", + "requires": { + "brace-expansion": "^2.0.1" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + } + } + }, "mongodb": { "version": "4.13.0", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz", diff --git a/services/web/app/src/Features/Uploads/FileTypeManager.js b/services/web/app/src/Features/Uploads/FileTypeManager.js index ed0a0b0def..15319bb180 100644 --- a/services/web/app/src/Features/Uploads/FileTypeManager.js +++ b/services/web/app/src/Features/Uploads/FileTypeManager.js @@ -3,41 +3,17 @@ const Path = require('path') const isUtf8 = require('utf-8-validate') const { promisifyAll } = require('../../util/promises') const Settings = require('@overleaf/settings') +const Minimatch = require('minimatch').Minimatch + +const fileIgnoreMatcher = new Minimatch(Settings.fileIgnorePattern, { + nocase: true, // make the whole path matching case-insensitive + // (previously we were only matching the extension case-insensitively but it seems safer to match the whole path) + dot: true, // allows matching on paths containing a dot e.g. /.git/foo/bar.txt +}) const FileTypeManager = { TEXT_EXTENSIONS: new Set(Settings.textExtensions.map(ext => `.${ext}`)), - IGNORE_EXTENSIONS: new Set([ - '.dvi', - '.aux', - '.log', - '.toc', - '.out', - '.pdfsync', - '.synctex', // synctex.gz is handled by the .gz extension - '.synctex(busy)', - '.fdb_latexmk', - '.fls', - // Index and glossary files - '.nlo', - '.ind', - '.glo', - '.gls', - '.glg', - // Bibtex - '.bbl', - '.blg', - // Misc/bad - '.doc', - '.docx', - '.gz', - '.swp', - ]), - - IGNORE_FILENAMES: new Set(['.gitignore']), - - IGNORE_FOLDERS: new Set(['__MACOSX', '.git', '.texpadtmp', '.R']), - MAX_TEXT_FILE_SIZE: 1 * 1024 * 1024, // 1 MB isDirectory(path, callback) { @@ -120,26 +96,10 @@ const FileTypeManager = { return true }, + // FIXME: we can convert this to a synchronous function if we want to shouldIgnore(path, callback) { - const basename = Path.basename(path) - const extension = Path.extname(basename).toLowerCase() - const folders = path.split('/') - let ignore = false - if (basename.startsWith('.') && basename !== '.latexmkrc') { - ignore = true - } - if (FileTypeManager.IGNORE_EXTENSIONS.has(extension)) { - ignore = true - } else if (FileTypeManager.IGNORE_FILENAMES.has(basename)) { - ignore = true - } else { - for (const folder of folders) { - if (FileTypeManager.IGNORE_FOLDERS.has(folder)) { - ignore = true - break - } - } - } + // use minimatch file matching to check if the path should be ignored + const ignore = fileIgnoreMatcher.match(path) callback(null, ignore) }, } diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 6a12236442..c95c48cab3 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -675,6 +675,10 @@ module.exports = { parseTextExtensions(process.env.ADDITIONAL_TEXT_EXTENSIONS) ), + fileIgnorePattern: + process.env.FILE_IGNORE_PATTERN || + '**/{{__MACOSX,.git,.texpadtmp,.R}{,/**},.!(latexmkrc),*.{dvi,aux,log,toc,out,pdfsync,synctex,synctex(busy),fdb_latexmk,fls,nlo,ind,glo,gls,glg,bbl,blg,doc,docx,gz,swp}}', + validRootDocExtensions: ['tex', 'Rtex', 'ltx'], emailConfirmationDisabled: diff --git a/services/web/package.json b/services/web/package.json index 6d56e9b052..9c2f35d00a 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -188,6 +188,7 @@ "mathjax": "^2.7.9", "mathjax-3": "npm:mathjax@^3.2.2", "method-override": "^2.3.3", + "minimatch": "^7.4.2", "minimist": "^1.2.7", "mmmagic": "^0.5.3", "moment": "^2.29.4", diff --git a/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js b/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js index d6495e2781..c2650955db 100644 --- a/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js +++ b/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js @@ -3,6 +3,7 @@ const { expect } = require('chai') const mockFs = require('mock-fs') const SandboxedModule = require('sandboxed-module') const { ObjectId } = require('mongodb') +const Settings = require('@overleaf/settings') const MODULE_PATH = '../../../../app/src/Features/Uploads/FileSystemImportManager.js' @@ -27,6 +28,7 @@ describe('FileSystemImportManager', function () { requires: { '@overleaf/settings': { textExtensions: ['tex', 'txt'], + fileIgnorePattern: Settings.fileIgnorePattern, // use the real pattern from the default settings }, '../Editor/EditorController': this.EditorController, },