diff --git a/package-lock.json b/package-lock.json index fe202b7d09..ffca801031 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4067,6 +4067,19 @@ "@lezer/lr": "^0.16.0" } }, + "node_modules/@lezer/generator": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-0.16.0.tgz", + "integrity": "sha512-dhkChgTGXdgj++0VV/Av1ARNe35Fj5uYuVt7W6g/7cDtHToS1dNn73rUtRb1cRO6kQXaISE/ne1cr3VofyBcng==", + "dev": true, + "dependencies": { + "@lezer/common": "^0.16.0", + "@lezer/lr": "^0.16.0" + }, + "bin": { + "lezer-generator": "dist/lezer-generator.cjs" + } + }, "node_modules/@lezer/highlight": { "version": "0.16.0", "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-0.16.0.tgz", @@ -35168,6 +35181,7 @@ "@babel/register": "^7.14.5", "@cypress/react": "^5.12.5", "@juggle/resize-observer": "^3.3.1", + "@lezer/generator": "^0.16.0", "@testing-library/cypress": "^8.0.3", "@testing-library/dom": "^8.13.0", "@testing-library/react": "^12.1.5", @@ -39606,6 +39620,16 @@ "@lezer/lr": "^0.16.0" } }, + "@lezer/generator": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-0.16.0.tgz", + "integrity": "sha512-dhkChgTGXdgj++0VV/Av1ARNe35Fj5uYuVt7W6g/7cDtHToS1dNn73rUtRb1cRO6kQXaISE/ne1cr3VofyBcng==", + "dev": true, + "requires": { + "@lezer/common": "^0.16.0", + "@lezer/lr": "^0.16.0" + } + }, "@lezer/highlight": { "version": "0.16.0", "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-0.16.0.tgz", @@ -42588,6 +42612,7 @@ "@cypress/react": "^5.12.5", "@juggle/resize-observer": "^3.3.1", "@lezer/common": "^0.16.0", + "@lezer/generator": "^0.16.0", "@lezer/highlight": "^0.16.0", "@lezer/lr": "^0.16.3", "@overleaf/logger": "^3.1.0", diff --git a/services/web/.eslintignore b/services/web/.eslintignore index 3d0f6c8d1a..af55608e46 100644 --- a/services/web/.eslintignore +++ b/services/web/.eslintignore @@ -4,3 +4,5 @@ modules/**/scripts frontend/js/vendor modules/**/frontend/js/vendor /public/ +modules/source-editor/frontend/js/lezer-latex/latex.mjs +modules/source-editor/frontend/js/lezer-latex/latex.terms.mjs diff --git a/services/web/.gitignore b/services/web/.gitignore index 061e7d6bc2..578ed2a2bd 100644 --- a/services/web/.gitignore +++ b/services/web/.gitignore @@ -92,4 +92,8 @@ cypress/downloads/ # Ace themes for conversion modules/source-editor/frontend/js/themes/ace/ +# Compiled parser files +modules/source-editor/frontend/js/lezer-latex/latex.mjs +modules/source-editor/frontend/js/lezer-latex/latex.terms.mjs + !**/fixtures/**/*.log diff --git a/services/web/.prettierignore b/services/web/.prettierignore index 04ecd4764c..f60bda48ce 100644 --- a/services/web/.prettierignore +++ b/services/web/.prettierignore @@ -6,3 +6,5 @@ modules/**/frontend/js/vendor public/js public/minjs frontend/stylesheets/components/nvd3.less +modules/source-editor/frontend/js/lezer-latex/latex.mjs +modules/source-editor/frontend/js/lezer-latex/latex.terms.mjs diff --git a/services/web/Dockerfile b/services/web/Dockerfile index a050ac79f0..94850a8ab6 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -29,6 +29,9 @@ FROM deps as dev COPY services/web /overleaf/services/web +# Build the latex parser +RUN cd /overleaf/services/web && npm run 'lezer-latex:generate' + RUN mkdir -p /overleaf/services/web/data/dumpFolder \ && mkdir -p /overleaf/services/web/data/logs \ && mkdir -p /overleaf/services/web/data/pdf \ diff --git a/services/web/package.json b/services/web/package.json index b061090a7e..d6b328924f 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -13,7 +13,7 @@ "test:unit:all": "npm run test:unit:run_dir -- test/unit/src modules/*/test/unit/src", "test:unit:all:silent": "npm run test:unit:all -- --reporter dot", "test:unit:app": "npm run test:unit:run_dir -- test/unit/src", - "test:frontend": "NODE_ENV=test TZ=GMT mocha --recursive --timeout 5000 --exit --extension js,ts,tsx --grep=$MOCHA_GREP --require test/frontend/bootstrap.js --ignore '**/*.spec.{js,ts,tsx}' test/frontend modules/*/test/frontend", + "test:frontend": "NODE_ENV=test TZ=GMT mocha --recursive --timeout 5000 --exit --extension js,mjs,ts,tsx --grep=$MOCHA_GREP --require test/frontend/bootstrap.js --ignore '**/*.spec.{js,ts,tsx}' test/frontend modules/*/test/frontend", "test:frontend:coverage": "c8 --all --include 'frontend/js' --include 'modules/*/frontend/js' --exclude 'frontend/js/vendor' --reporter=lcov --reporter=text-summary npm run test:frontend", "test:karma": "karma start", "test:karma:single": "karma start --no-auto-watch --single-run", @@ -34,6 +34,8 @@ "convert-themes": "node modules/source-editor/frontend/js/themes/convert.js", "cypress:open-ct": "SHARELATEX_CONFIG=$PWD/config/settings.webpack.js cypress open --component", "cypress:run-ct": "SHARELATEX_CONFIG=$PWD/config/settings.webpack.js cypress run --component", + "lezer-latex:generate": "node modules/source-editor/scripts/lezer-latex/generate.js", + "lezer-latex:run": "node modules/source-editor/scripts/lezer-latex/run.mjs", "routes": "bin/routes" }, "browserslist": [ @@ -205,6 +207,7 @@ "@babel/register": "^7.14.5", "@cypress/react": "^5.12.5", "@juggle/resize-observer": "^3.3.1", + "@lezer/generator": "^0.16.0", "@testing-library/cypress": "^8.0.3", "@testing-library/dom": "^8.13.0", "@testing-library/react": "^12.1.5", diff --git a/services/web/webpack-plugins/lezer-grammar-compiler.js b/services/web/webpack-plugins/lezer-grammar-compiler.js new file mode 100644 index 0000000000..6adf3ec2a2 --- /dev/null +++ b/services/web/webpack-plugins/lezer-grammar-compiler.js @@ -0,0 +1,49 @@ +const { + compile, + options, +} = require('../modules/source-editor/scripts/lezer-latex/generate') +const fs = require('fs') + +const PLUGIN_NAME = 'lezer-grammar-compiler' + +class LezerGrammarCompilerPlugin { + apply(compiler) { + compiler.hooks.make.tap(PLUGIN_NAME, compilation => { + // Add the grammar file to the file paths watched by webpack + compilation.fileDependencies.add(options.grammarPath) + }) + compiler.hooks.beforeCompile.tapAsync( + PLUGIN_NAME, + (_compilation, callback) => { + // Check timestamps on grammar and parser files, and re-compile if needed. + // (Note: the compiled parser file is watched by webpack, and so will trigger + // a second compilation immediately after. This seems harmless.) + if ( + !fs.existsSync(options.parserOutputPath) || + !fs.existsSync(options.termsOutputPath) + ) { + console.log('Parser does not exist, compiling') + compile() + return callback() + } + fs.stat(options.grammarPath, (err, grammarStat) => { + if (err) { + return callback(err) + } + fs.stat(options.parserOutputPath, (err, parserStat) => { + if (err) { + return callback(err) + } + callback() + if (grammarStat.mtime > parserStat.mtime) { + console.log('Grammar file newer than parser file, re-compiling') + compile() + } + }) + }) + } + ) + } +} + +module.exports = { LezerGrammarCompilerPlugin } diff --git a/services/web/webpack.config.js b/services/web/webpack.config.js index 3a6462b6fa..626d1214e3 100644 --- a/services/web/webpack.config.js +++ b/services/web/webpack.config.js @@ -4,6 +4,9 @@ const webpack = require('webpack') const CopyPlugin = require('copy-webpack-plugin') const WebpackAssetsManifest = require('webpack-assets-manifest') const MiniCssExtractPlugin = require('mini-css-extract-plugin') +const { + LezerGrammarCompilerPlugin, +} = require('./webpack-plugins/lezer-grammar-compiler') const PackageVersions = require('./app/src/infrastructure/PackageVersions') @@ -226,6 +229,8 @@ module.exports = { }, plugins: [ + new LezerGrammarCompilerPlugin(), + // Generate a manifest.json file which is used by the backend to map the // base filenames to the generated output filenames new WebpackAssetsManifest({