Merge pull request #12195 from overleaf/bg-use-glob-ignore

Change ignored file matching to use standard `minimatch` module

GitOrigin-RevId: ef8f13c039015e7bb9894441fb6170703941653c

services/web/app/src/Features/Uploads/FileTypeManager.js

@@ -3,41 +3,17 @@ const Path = require('path')
 const isUtf8 = require('utf-8-validate')
 const { promisifyAll } = require('../../util/promises')
 const Settings = require('@overleaf/settings')
+const Minimatch = require('minimatch').Minimatch
+
+const fileIgnoreMatcher = new Minimatch(Settings.fileIgnorePattern, {
+  nocase: true, // make the whole path matching case-insensitive
+  // (previously we were only matching the extension case-insensitively but it seems safer to match the whole path)
+  dot: true, // allows matching on paths containing a dot e.g. /.git/foo/bar.txt
+})
 
 const FileTypeManager = {
   TEXT_EXTENSIONS: new Set(Settings.textExtensions.map(ext => `.${ext}`)),
 
-  IGNORE_EXTENSIONS: new Set([
-    '.dvi',
-    '.aux',
-    '.log',
-    '.toc',
-    '.out',
-    '.pdfsync',
-    '.synctex', // synctex.gz is handled by the .gz extension
-    '.synctex(busy)',
-    '.fdb_latexmk',
-    '.fls',
-    // Index and glossary files
-    '.nlo',
-    '.ind',
-    '.glo',
-    '.gls',
-    '.glg',
-    // Bibtex
-    '.bbl',
-    '.blg',
-    // Misc/bad
-    '.doc',
-    '.docx',
-    '.gz',
-    '.swp',
-  ]),
-
-  IGNORE_FILENAMES: new Set(['.gitignore']),
-
-  IGNORE_FOLDERS: new Set(['__MACOSX', '.git', '.texpadtmp', '.R']),
-
   MAX_TEXT_FILE_SIZE: 1 * 1024 * 1024, // 1 MB
 
   isDirectory(path, callback) {
@@ -120,26 +96,10 @@ const FileTypeManager = {
     return true
   },
 
+  // FIXME: we can convert this to a synchronous function if we want to
   shouldIgnore(path, callback) {
-    const basename = Path.basename(path)
-    const extension = Path.extname(basename).toLowerCase()
-    const folders = path.split('/')
-    let ignore = false
-    if (basename.startsWith('.') && basename !== '.latexmkrc') {
-      ignore = true
-    }
-    if (FileTypeManager.IGNORE_EXTENSIONS.has(extension)) {
-      ignore = true
-    } else if (FileTypeManager.IGNORE_FILENAMES.has(basename)) {
-      ignore = true
-    } else {
-      for (const folder of folders) {
-        if (FileTypeManager.IGNORE_FOLDERS.has(folder)) {
-          ignore = true
-          break
-        }
-      }
-    }
+    // use minimatch file matching to check if the path should be ignored
+    const ignore = fileIgnoreMatcher.match(path)
     callback(null, ignore)
   },
 }

services/web/config/settings.defaults.js

@@ -675,6 +675,10 @@ module.exports = {
     parseTextExtensions(process.env.ADDITIONAL_TEXT_EXTENSIONS)
   ),
 
+  fileIgnorePattern:
+    process.env.FILE_IGNORE_PATTERN ||
+    '**/{{__MACOSX,.git,.texpadtmp,.R}{,/**},.!(latexmkrc),*.{dvi,aux,log,toc,out,pdfsync,synctex,synctex(busy),fdb_latexmk,fls,nlo,ind,glo,gls,glg,bbl,blg,doc,docx,gz,swp}}',
+
   validRootDocExtensions: ['tex', 'Rtex', 'ltx'],
 
   emailConfirmationDisabled:

services/web/package.json

@@ -188,6 +188,7 @@
     "mathjax": "^2.7.9",
     "mathjax-3": "npm:mathjax@^3.2.2",
     "method-override": "^2.3.3",
+    "minimatch": "^7.4.2",
     "minimist": "^1.2.7",
     "mmmagic": "^0.5.3",
     "moment": "^2.29.4",

services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js

@@ -3,6 +3,7 @@ const { expect } = require('chai')
 const mockFs = require('mock-fs')
 const SandboxedModule = require('sandboxed-module')
 const { ObjectId } = require('mongodb')
+const Settings = require('@overleaf/settings')
 
 const MODULE_PATH =
   '../../../../app/src/Features/Uploads/FileSystemImportManager.js'
@@ -27,6 +28,7 @@ describe('FileSystemImportManager', function () {
       requires: {
         '@overleaf/settings': {
           textExtensions: ['tex', 'txt'],
+          fileIgnorePattern: Settings.fileIgnorePattern, // use the real pattern from the default settings
         },
         '../Editor/EditorController': this.EditorController,
       },