From 67e7621633abefdedf5518555f3198ac10daa662 Mon Sep 17 00:00:00 2001 From: Mathias Jakobsen Date: Mon, 3 Jul 2023 12:18:27 +0200 Subject: [PATCH] Merge pull request #13572 from overleaf/mj-bibtex-grammar [cm6] Add support for bibtex GitOrigin-RevId: 28bc8e47c53df1612c1e30cf690e893b0bbf500c --- package-lock.json | 88 ++++++++--------- services/web/.eslintignore | 2 + services/web/.gitignore | 2 + services/web/.prettierignore | 2 + .../languages/bibtex/bibtex-language.ts | 10 ++ .../languages/bibtex/completions/snippets.ts | 95 +++++++++++++++++++ .../source-editor/languages/bibtex/index.ts | 8 ++ .../features/source-editor/languages/index.ts | 8 +- .../source-editor/lezer-bibtex/bibtex.grammar | 85 +++++++++++++++++ .../source-editor/lezer-bibtex/highlight.mjs | 15 +++ .../source-editor/source-editor.stories.tsx | 64 +++++++++++++ services/web/package.json | 10 +- services/web/scripts/lezer-latex/generate.js | 52 ++++++---- services/web/scripts/lezer-latex/run.mjs | 7 +- .../webpack-plugins/lezer-grammar-compiler.js | 66 +++++++------ 15 files changed, 413 insertions(+), 101 deletions(-) create mode 100644 services/web/frontend/js/features/source-editor/languages/bibtex/bibtex-language.ts create mode 100644 services/web/frontend/js/features/source-editor/languages/bibtex/completions/snippets.ts create mode 100644 services/web/frontend/js/features/source-editor/languages/bibtex/index.ts create mode 100644 services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar create mode 100644 services/web/frontend/js/features/source-editor/lezer-bibtex/highlight.mjs diff --git a/package-lock.json b/package-lock.json index e6027c5bcc..a51df4ce84 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5796,9 +5796,9 @@ "devOptional": true }, "node_modules/@lezer/common": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.0.2.tgz", - "integrity": "sha512-SVgiGtMnMnW3ActR8SXgsDhw7a0w0ChHSYAyAUxxrOiJ1OqYWEKk/xJd84tTSPo1mo6DXLObAJALNnd0Hrv7Ng==" + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.0.3.tgz", + "integrity": "sha512-JH4wAXCgUOcCGNekQPLhVeUtIqjH0yPBs7vvUdSjyQama9618IOKFJwkv2kcqdhF0my8hQEgCTEJU0GIgnahvA==" }, "node_modules/@lezer/css": { "version": "1.0.0", @@ -5810,22 +5810,22 @@ } }, "node_modules/@lezer/generator": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.1.3.tgz", - "integrity": "sha512-qGF0I2TTJ+VBjjsVX8FGqKJy3laALBnVbD5EbXEu13Sgszl/vjnxjcZ69O8w9IK8/WtVFQLspU4UjCCUNRlWzA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.3.0.tgz", + "integrity": "sha512-7HfulDoOMOkskb97fnwgpC6StwPVSob4ptc0iuOH72rapNQBbp6lVj05y7vc5IM0E9pjFjiLmNQeiBiSbLpCtA==", "dev": true, "dependencies": { - "@lezer/common": "^1.0.0", - "@lezer/lr": "^1.0.0" + "@lezer/common": "^1.0.2", + "@lezer/lr": "^1.3.0" }, "bin": { "lezer-generator": "dist/lezer-generator.cjs" } }, "node_modules/@lezer/highlight": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.1.3.tgz", - "integrity": "sha512-3vLKLPThO4td43lYRBygmMY18JN3CPh9w+XS2j8WC30vR4yZeFG4z1iFe4jXE43NtGqe//zHW5q8ENLlHvz9gw==", + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.1.6.tgz", + "integrity": "sha512-cmSJYa2us+r3SePpRCjN5ymCqCPv+zyXmDl0ciWtVaNiORT/MxM7ZgOMQZADD0o51qOaOg24qc/zBViOIwAjJg==", "dependencies": { "@lezer/common": "^1.0.0" } @@ -5849,17 +5849,17 @@ } }, "node_modules/@lezer/lr": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.3.3.tgz", - "integrity": "sha512-JPQe3mwJlzEVqy67iQiiGozhcngbO8QBgpqZM6oL1Wj/dXckrEexpBLeFkq0edtW5IqnPRFxA24BHJni8Js69w==", + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.3.7.tgz", + "integrity": "sha512-ssHKb3p0MxhJXT2i7UBmgAY1BIM3Uq/D772Qutu3EVmxWIyNMU12nQ0rL3Fhu+MiFtiTzyTmd3xGwEf3ON5PSA==", "dependencies": { "@lezer/common": "^1.0.0" } }, "node_modules/@lezer/markdown": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@lezer/markdown/-/markdown-1.0.2.tgz", - "integrity": "sha512-8CY0OoZ6V5EzPjSPeJ4KLVbtXdLBd8V6sRCooN5kHnO28ytreEGTyrtU/zUwo/XLRzGr/e1g44KlzKi3yWGB5A==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@lezer/markdown/-/markdown-1.0.3.tgz", + "integrity": "sha512-QEcXFCKf1TBdVhmxL2V9afJTIs4w795DTl2NKnsYZyMOtMsA+5AlEy0biPo/Ojv05ELkk6HIPSDBj0g+ShlkBw==", "dependencies": { "@lezer/common": "^1.0.0", "@lezer/highlight": "^1.0.0" @@ -41266,10 +41266,10 @@ "@contentful/rich-text-html-renderer": "^16.0.2", "@contentful/rich-text-types": "^16.0.2", "@google-cloud/bigquery": "^6.0.1", - "@lezer/common": "^1.0.2", - "@lezer/highlight": "^1.1.3", - "@lezer/lr": "^1.3.3", - "@lezer/markdown": "^1.0.2", + "@lezer/common": "^1.0.3", + "@lezer/highlight": "^1.1.6", + "@lezer/lr": "^1.3.7", + "@lezer/markdown": "^1.0.3", "@node-oauth/oauth2-server": "^4.3.0", "@opentelemetry/api": "^1.0.4", "@opentelemetry/auto-instrumentations-web": "^0.27.2", @@ -41441,7 +41441,7 @@ "devDependencies": { "@babel/register": "^7.21.0", "@juggle/resize-observer": "^3.3.1", - "@lezer/generator": "^1.1.3", + "@lezer/generator": "^1.3.0", "@testing-library/cypress": "^9.0.0", "@testing-library/dom": "^9.3.0", "@testing-library/react": "^12.1.5", @@ -46835,9 +46835,9 @@ "devOptional": true }, "@lezer/common": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.0.2.tgz", - "integrity": "sha512-SVgiGtMnMnW3ActR8SXgsDhw7a0w0ChHSYAyAUxxrOiJ1OqYWEKk/xJd84tTSPo1mo6DXLObAJALNnd0Hrv7Ng==" + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.0.3.tgz", + "integrity": "sha512-JH4wAXCgUOcCGNekQPLhVeUtIqjH0yPBs7vvUdSjyQama9618IOKFJwkv2kcqdhF0my8hQEgCTEJU0GIgnahvA==" }, "@lezer/css": { "version": "1.0.0", @@ -46849,19 +46849,19 @@ } }, "@lezer/generator": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.1.3.tgz", - "integrity": "sha512-qGF0I2TTJ+VBjjsVX8FGqKJy3laALBnVbD5EbXEu13Sgszl/vjnxjcZ69O8w9IK8/WtVFQLspU4UjCCUNRlWzA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.3.0.tgz", + "integrity": "sha512-7HfulDoOMOkskb97fnwgpC6StwPVSob4ptc0iuOH72rapNQBbp6lVj05y7vc5IM0E9pjFjiLmNQeiBiSbLpCtA==", "dev": true, "requires": { - "@lezer/common": "^1.0.0", - "@lezer/lr": "^1.0.0" + "@lezer/common": "^1.0.2", + "@lezer/lr": "^1.3.0" } }, "@lezer/highlight": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.1.3.tgz", - "integrity": "sha512-3vLKLPThO4td43lYRBygmMY18JN3CPh9w+XS2j8WC30vR4yZeFG4z1iFe4jXE43NtGqe//zHW5q8ENLlHvz9gw==", + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.1.6.tgz", + "integrity": "sha512-cmSJYa2us+r3SePpRCjN5ymCqCPv+zyXmDl0ciWtVaNiORT/MxM7ZgOMQZADD0o51qOaOg24qc/zBViOIwAjJg==", "requires": { "@lezer/common": "^1.0.0" } @@ -46885,17 +46885,17 @@ } }, "@lezer/lr": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.3.3.tgz", - "integrity": "sha512-JPQe3mwJlzEVqy67iQiiGozhcngbO8QBgpqZM6oL1Wj/dXckrEexpBLeFkq0edtW5IqnPRFxA24BHJni8Js69w==", + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.3.7.tgz", + "integrity": "sha512-ssHKb3p0MxhJXT2i7UBmgAY1BIM3Uq/D772Qutu3EVmxWIyNMU12nQ0rL3Fhu+MiFtiTzyTmd3xGwEf3ON5PSA==", "requires": { "@lezer/common": "^1.0.0" } }, "@lezer/markdown": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@lezer/markdown/-/markdown-1.0.2.tgz", - "integrity": "sha512-8CY0OoZ6V5EzPjSPeJ4KLVbtXdLBd8V6sRCooN5kHnO28ytreEGTyrtU/zUwo/XLRzGr/e1g44KlzKi3yWGB5A==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@lezer/markdown/-/markdown-1.0.3.tgz", + "integrity": "sha512-QEcXFCKf1TBdVhmxL2V9afJTIs4w795DTl2NKnsYZyMOtMsA+5AlEy0biPo/Ojv05ELkk6HIPSDBj0g+ShlkBw==", "requires": { "@lezer/common": "^1.0.0", "@lezer/highlight": "^1.0.0" @@ -50249,11 +50249,11 @@ "@contentful/rich-text-types": "^16.0.2", "@google-cloud/bigquery": "^6.0.1", "@juggle/resize-observer": "^3.3.1", - "@lezer/common": "^1.0.2", - "@lezer/generator": "^1.1.3", - "@lezer/highlight": "^1.1.3", - "@lezer/lr": "^1.3.3", - "@lezer/markdown": "^1.0.2", + "@lezer/common": "^1.0.3", + "@lezer/generator": "^1.3.0", + "@lezer/highlight": "^1.1.6", + "@lezer/lr": "^1.3.7", + "@lezer/markdown": "^1.0.3", "@node-oauth/oauth2-server": "^4.3.0", "@opentelemetry/api": "^1.0.4", "@opentelemetry/auto-instrumentations-web": "^0.27.2", diff --git a/services/web/.eslintignore b/services/web/.eslintignore index 3b4343dcca..a76296621c 100644 --- a/services/web/.eslintignore +++ b/services/web/.eslintignore @@ -6,3 +6,5 @@ modules/**/frontend/js/vendor /public/ frontend/js/features/source-editor/lezer-latex/latex.mjs frontend/js/features/source-editor/lezer-latex/latex.terms.mjs +frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs +frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs diff --git a/services/web/.gitignore b/services/web/.gitignore index f49076694e..801187d4a4 100644 --- a/services/web/.gitignore +++ b/services/web/.gitignore @@ -94,5 +94,7 @@ frontend/js/features/source-editor/themes/ace/ # Compiled parser files frontend/js/features/source-editor/lezer-latex/latex.mjs frontend/js/features/source-editor/lezer-latex/latex.terms.mjs +frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs +frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs !**/fixtures/**/*.log diff --git a/services/web/.prettierignore b/services/web/.prettierignore index bd3c471b9e..777a6c225b 100644 --- a/services/web/.prettierignore +++ b/services/web/.prettierignore @@ -8,3 +8,5 @@ public/minjs frontend/stylesheets/components/nvd3.less frontend/js/features/source-editor/lezer-latex/latex.mjs frontend/js/features/source-editor/lezer-latex/latex.terms.mjs +frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs +frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs diff --git a/services/web/frontend/js/features/source-editor/languages/bibtex/bibtex-language.ts b/services/web/frontend/js/features/source-editor/languages/bibtex/bibtex-language.ts new file mode 100644 index 0000000000..e0f9d45d9a --- /dev/null +++ b/services/web/frontend/js/features/source-editor/languages/bibtex/bibtex-language.ts @@ -0,0 +1,10 @@ +import { LRLanguage } from '@codemirror/language' +import { parser } from '../../lezer-bibtex/bibtex.mjs' +import { bibtexEntryCompletions } from './completions/snippets' + +export const BibTeXLanguage = LRLanguage.define({ + parser, + languageData: { + autocomplete: bibtexEntryCompletions, + }, +}) diff --git a/services/web/frontend/js/features/source-editor/languages/bibtex/completions/snippets.ts b/services/web/frontend/js/features/source-editor/languages/bibtex/completions/snippets.ts new file mode 100644 index 0000000000..4722ed0448 --- /dev/null +++ b/services/web/frontend/js/features/source-editor/languages/bibtex/completions/snippets.ts @@ -0,0 +1,95 @@ +import { CompletionContext, snippet } from '@codemirror/autocomplete' + +type Environment = { + name: string + requiredAttributes: string[] +} + +const environments: Environment[] = [ + { + name: 'article', + requiredAttributes: ['author', 'title', 'journal', 'year'], + }, + { + name: 'book', + requiredAttributes: ['author', 'title', 'publisher', 'year'], + }, + { + name: 'booklet', + requiredAttributes: ['key', 'title'], + }, + { + name: 'conference', + requiredAttributes: ['key', 'title', 'year'], + }, + { + name: 'inbook', + requiredAttributes: ['author', 'title', 'publisher', 'year', 'chapter'], + }, + { + name: 'incollection', + requiredAttributes: ['author', 'title', 'booktitle', 'publisher', 'year'], + }, + { + name: 'inproceedings', + requiredAttributes: ['author', 'title', 'booktitle', 'year'], + }, + { + name: 'manual', + requiredAttributes: ['key', 'title'], + }, + { + name: 'masterthesis', + requiredAttributes: ['author', 'title', 'school', 'year'], + }, + { + name: 'misc', + requiredAttributes: ['key', 'note'], + }, + { + name: 'phdthesis', + requiredAttributes: ['author', 'title', 'school', 'year'], + }, + { + name: 'proceedings', + requiredAttributes: ['key', 'title', 'year'], + }, + { + name: 'techreport', + requiredAttributes: ['author', 'title', 'institution', 'year'], + }, + { + name: 'unpublished', + requiredAttributes: ['author', 'title', 'note'], + }, +] + +const prepareSnippet = (environment: Environment) => { + return `@${ + environment.name + }{#{citation-key},${environment.requiredAttributes.map( + attribute => ` + ${attribute} = #{}` + )} +}` +} + +export function bibtexEntryCompletions(context: CompletionContext) { + const word = context.matchBefore(/@\w*/) + if (word?.from === word?.to && !context.explicit) return null + return { + from: word?.from ?? context.pos, + options: [ + ...environments.map(env => ({ + label: `@${env.name}`, + type: 'snippet', + apply: snippet(prepareSnippet(env)), + })), + { + label: '@string', + type: 'snippet', + apply: snippet('@string{#{string-key} = #{}}'), + }, + ], + } +} diff --git a/services/web/frontend/js/features/source-editor/languages/bibtex/index.ts b/services/web/frontend/js/features/source-editor/languages/bibtex/index.ts new file mode 100644 index 0000000000..74c30f301c --- /dev/null +++ b/services/web/frontend/js/features/source-editor/languages/bibtex/index.ts @@ -0,0 +1,8 @@ +import { LanguageSupport, indentUnit } from '@codemirror/language' +import { BibTeXLanguage } from './bibtex-language' + +export const bibtex = () => { + return new LanguageSupport(BibTeXLanguage, [ + indentUnit.of(' '), // 4 spaces + ]) +} diff --git a/services/web/frontend/js/features/source-editor/languages/index.ts b/services/web/frontend/js/features/source-editor/languages/index.ts index 9f03fa2aa6..bee9b757c3 100644 --- a/services/web/frontend/js/features/source-editor/languages/index.ts +++ b/services/web/frontend/js/features/source-editor/languages/index.ts @@ -5,7 +5,6 @@ export const languages = [ name: 'latex', extensions: [ 'tex', - 'bib', 'sty', 'cls', 'clo', @@ -50,6 +49,13 @@ export const languages = [ return import('./latex').then(m => m.latex()) }, }), + LanguageDescription.of({ + name: 'bibtex', + extensions: ['bib'], + load: () => { + return import('./bibtex').then(m => m.bibtex()) + }, + }), LanguageDescription.of({ name: 'markdown', extensions: ['md', 'markdown'], diff --git a/services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar b/services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar new file mode 100644 index 0000000000..e44f02b863 --- /dev/null +++ b/services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar @@ -0,0 +1,85 @@ + + +@top Bibliography { + (Declaration | StringDeclaration)* +} + +@tokens { + whiteSpace { @whitespace+ } + Identifier { $[a-zA-Z:_0-9-]+ } + StringName { $[a-zA-Z:_] $[a-zA-Z:_0-9-]* } + FieldName {$[a-zA-Z-_0-9]+} + LiteralString { + '"' (!["] | "\\" _)* '"'? + } + EntryTypeName { $[a-zA-Z]+ } + Number { @digit+ } + StringKeyword {"@"$[Ss]$[Tt]$[Rr]$[Ii]$[Nn]$[Gg]} + "@" "{" "}" "\"" "," "#" "@string" + Comment { "%" ![\n]* } +} + +// FIXME: Technically skipping comments is wrong here. They can only appear +// alone on a line, but I'm not sure how to express that easily in Lezer +@skip {whiteSpace | Comment} + +StringDeclaration { + StringKeyword "{" + Field* + "}" +} + +Declaration { + EntryName { "@" EntryTypeName } "{" + Identifier + + fieldEntry { + ("," Field ) + }* + ("," )? + "}" +} + +Field { + Name "=" Expression +} + +Expression { + BracedString | + Number | + StringConcatenation +} + +@local tokens { + openBracedContents {"{"} + closeBracedContents {"}"} + @else nonClosingBracedContents +} + +@skip {}{ + bracedStringContents { + ( + nonClosingBracedContents | + nestedBracedString { + openBracedContents + bracedStringContents + closeBracedContents + } + )* +} + BracedString { + "{" + bracedStringContents closeBracedContents + } +} + +// TODO: Implement this +@precedence { concatenation @left } + +StringConcatenation { + StringConcatenation !concatenation "#" StringConcatenation | + LiteralString | + StringName +} + +@external propSource highlighting from "./highlight.mjs" \ No newline at end of file diff --git a/services/web/frontend/js/features/source-editor/lezer-bibtex/highlight.mjs b/services/web/frontend/js/features/source-editor/lezer-bibtex/highlight.mjs new file mode 100644 index 0000000000..9310a4deb7 --- /dev/null +++ b/services/web/frontend/js/features/source-editor/lezer-bibtex/highlight.mjs @@ -0,0 +1,15 @@ +import { styleTags, tags as t } from '@lezer/highlight' + +export const highlighting = styleTags({ + LiteralString: t.string, + 'BracedString/...': t.string, + Number: t.number, + Identifier: t.name, + 'EntryName/...': t.keyword, + FieldName: t.attributeName, + Expression: t.attributeValue, + '#': t.operator, + StringKeyword: t.keyword, + StringName: t.variableName, + Comment: t.comment, +}) diff --git a/services/web/frontend/stories/source-editor/source-editor.stories.tsx b/services/web/frontend/stories/source-editor/source-editor.stories.tsx index 9b62719a3b..a1eca33196 100644 --- a/services/web/frontend/stories/source-editor/source-editor.stories.tsx +++ b/services/web/frontend/stories/source-editor/source-editor.stories.tsx @@ -103,6 +103,21 @@ export const Visual = (args: any, { globals: { theme } }: any) => { return } +export const Bibtex = (args: any, { globals: { theme } }: any) => { + useScope({ + editor: { + sharejs_doc: mockDoc(content.bib, changes.bib), + open_doc_name: 'example.bib', + }, + settings: { + ...settings, + overallTheme: theme === 'default-' ? '' : theme, + }, + }) + + return +} + const MAX_DOC_LENGTH = 2 * 1024 * 1024 // window.maxDocLength const mockDoc = (content: string, changes: Array> = []) => { @@ -163,6 +178,7 @@ const changes: Record>> = { }, ], md: [], + bib: [], } const content = { @@ -290,4 +306,52 @@ We hope you find Overleaf useful, and do take a look at our \\href{https://www.o This is **bold** This is _italic_`, + bib: `@book{texbook, + author = {Donald E. Knuth}, + year = {1986}, + title = {The {\\TeX} Book}, + publisher = {Addison-Wesley Professional} +} + +@book{latex:companion, + author = {Frank Mittelbach and Michel Gossens + and Johannes Braams and David Carlisle + and Chris Rowley}, + year = {2004}, + title = {The {\\LaTeX} Companion}, + publisher = {Addison-Wesley Professional}, + edition = {2} +} + +@book{latex2e, + author = {Leslie Lamport}, + year = {1994}, + title = {{\\LaTeX}: a Document Preparation System}, + publisher = {Addison Wesley}, + address = {Massachusetts}, + edition = {2} +} + +@article{knuth:1984, + title={Literate Programming}, + author={Donald E. Knuth}, + journal={The Computer Journal}, + volume={27}, + number={2}, + pages={97--111}, + year={1984}, + publisher={Oxford University Press} +} + +@inproceedings{lesk:1977, + title={Computer Typesetting of Technical Journals on {UNIX}}, + author={Michael Lesk and Brian Kernighan}, + booktitle={Proceedings of American Federation of + Information Processing Societies: 1977 + National Computer Conference}, + pages={879--888}, + year={1977}, + address={Dallas, Texas} +} +`, } diff --git a/services/web/package.json b/services/web/package.json index 34c5ec6000..1fae1bcbb9 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -82,10 +82,10 @@ "@contentful/rich-text-html-renderer": "^16.0.2", "@contentful/rich-text-types": "^16.0.2", "@google-cloud/bigquery": "^6.0.1", - "@lezer/common": "^1.0.2", - "@lezer/highlight": "^1.1.3", - "@lezer/lr": "^1.3.3", - "@lezer/markdown": "^1.0.2", + "@lezer/common": "^1.0.3", + "@lezer/highlight": "^1.1.6", + "@lezer/lr": "^1.3.7", + "@lezer/markdown": "^1.0.3", "@node-oauth/oauth2-server": "^4.3.0", "@opentelemetry/api": "^1.0.4", "@opentelemetry/auto-instrumentations-web": "^0.27.2", @@ -257,7 +257,7 @@ "devDependencies": { "@babel/register": "^7.21.0", "@juggle/resize-observer": "^3.3.1", - "@lezer/generator": "^1.1.3", + "@lezer/generator": "^1.3.0", "@testing-library/cypress": "^9.0.0", "@testing-library/dom": "^9.3.0", "@testing-library/react": "^12.1.5", diff --git a/services/web/scripts/lezer-latex/generate.js b/services/web/scripts/lezer-latex/generate.js index d7fd6d8412..b2e73ffc04 100644 --- a/services/web/scripts/lezer-latex/generate.js +++ b/services/web/scripts/lezer-latex/generate.js @@ -2,23 +2,39 @@ const { buildParserFile } = require('@lezer/generator') const { writeFileSync, readFileSync } = require('fs') const path = require('path') -const options = { - grammarPath: path.resolve( - __dirname, - '../../frontend/js/features/source-editor/lezer-latex/latex.grammar' - ), - parserOutputPath: path.resolve( - __dirname, - '../../frontend/js/features/source-editor/lezer-latex/latex.mjs' - ), - termsOutputPath: path.resolve( - __dirname, - '../../frontend/js/features/source-editor/lezer-latex/latex.terms.mjs' - ), -} +const grammars = [ + { + grammarPath: path.resolve( + __dirname, + '../../frontend/js/features/source-editor/lezer-latex/latex.grammar' + ), + parserOutputPath: path.resolve( + __dirname, + '../../frontend/js/features/source-editor/lezer-latex/latex.mjs' + ), + termsOutputPath: path.resolve( + __dirname, + '../../frontend/js/features/source-editor/lezer-latex/latex.terms.mjs' + ), + }, + { + grammarPath: path.resolve( + __dirname, + '../../frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar' + ), + parserOutputPath: path.resolve( + __dirname, + '../../frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs' + ), + termsOutputPath: path.resolve( + __dirname, + '../../frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs' + ), + }, +] -function compile() { - const { grammarPath, termsOutputPath, parserOutputPath } = options +function compile(grammar) { + const { grammarPath, termsOutputPath, parserOutputPath } = grammar const moduleStyle = 'es' console.info(`Compiling ${grammarPath}`) @@ -40,11 +56,11 @@ function compile() { console.info('Done!') } -module.exports = { compile, options } +module.exports = { compile, grammars } if (require.main === module) { try { - compile() + grammars.forEach(compile) process.exit(0) } catch (err) { console.error(err) diff --git a/services/web/scripts/lezer-latex/run.mjs b/services/web/scripts/lezer-latex/run.mjs index a16731a8bc..d25d9887dc 100644 --- a/services/web/scripts/lezer-latex/run.mjs +++ b/services/web/scripts/lezer-latex/run.mjs @@ -1,11 +1,13 @@ import { readFileSync } from 'fs' import { logTree } from '../../frontend/js/features/source-editor/lezer-latex/print-tree.mjs' -import { parser } from '../../frontend/js/features/source-editor/lezer-latex/latex.mjs' +import { parser as LaTeXParser } from '../../frontend/js/features/source-editor/lezer-latex/latex.mjs' +import { parser as BibTeXParser } from '../../frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs' -// Runs the lezer-latex parser on a supplied file, and prints the resulting +// Runs the lezer-latex or lezer-bibtex parser on a supplied file, and prints the resulting // parse tree to stdout // // show parse tree: lezer-latex-run.js test/frontend/shared/lezer-latex/examples/amsmath.tex +// lezer-latex-run.js test/frontend/shared/lezer-latex/examples/overleaf.bib // show error summary: lezer-latex-run.js coverage test/frontend/shared/lezer-latex/examples/amsmath.tex let files = process.argv.slice(2) @@ -27,6 +29,7 @@ function reportErrorCounts(output) { function parseFile(filename) { const text = readFileSync(filename).toString() const t0 = process.hrtime() + const parser = filename.endsWith('.bib') ? BibTeXParser : LaTeXParser const tree = parser.parse(text) const dt = process.hrtime(t0) const timeTaken = dt[0] + dt[1] * 1e-9 diff --git a/services/web/webpack-plugins/lezer-grammar-compiler.js b/services/web/webpack-plugins/lezer-grammar-compiler.js index efadc5e542..3544116dc6 100644 --- a/services/web/webpack-plugins/lezer-grammar-compiler.js +++ b/services/web/webpack-plugins/lezer-grammar-compiler.js @@ -4,45 +4,49 @@ const modulePath = path.resolve(__dirname, '../scripts/lezer-latex/generate.js') try { fs.accessSync(modulePath, fs.constants.W_OK) - const { compile, options } = require(modulePath) + const { compile, grammars } = require(modulePath) const PLUGIN_NAME = 'lezer-grammar-compiler' class LezerGrammarCompilerPlugin { apply(compiler) { - compiler.hooks.make.tap(PLUGIN_NAME, compilation => { - // Add the grammar file to the file paths watched by webpack - compilation.fileDependencies.add(options.grammarPath) - }) - compiler.hooks.beforeCompile.tapAsync( - PLUGIN_NAME, - (_compilation, callback) => { - // Check timestamps on grammar and parser files, and re-compile if needed. - // (Note: the compiled parser file is watched by webpack, and so will trigger - // a second compilation immediately after. This seems harmless.) - if ( - !fs.existsSync(options.parserOutputPath) || - !fs.existsSync(options.termsOutputPath) - ) { - console.log('Parser does not exist, compiling') - compile() - return callback() - } - fs.stat(options.grammarPath, (err, grammarStat) => { - if (err) { - return callback(err) + for (const grammar of grammars) { + compiler.hooks.make.tap(PLUGIN_NAME, compilation => { + // Add the grammar file to the file paths watched by webpack + compilation.fileDependencies.add(grammar.grammarPath) + }) + compiler.hooks.beforeCompile.tapAsync( + PLUGIN_NAME, + (_compilation, callback) => { + // Check timestamps on grammar and parser files, and re-compile if needed. + // (Note: the compiled parser file is watched by webpack, and so will trigger + // a second compilation immediately after. This seems harmless.) + if ( + !fs.existsSync(grammar.parserOutputPath) || + !fs.existsSync(grammar.termsOutputPath) + ) { + console.log('Parser does not exist, compiling') + compile(grammar) + return callback() } - fs.stat(options.parserOutputPath, (err, parserStat) => { + fs.stat(grammar.grammarPath, (err, grammarStat) => { if (err) { return callback(err) } - callback() - if (grammarStat.mtime > parserStat.mtime) { - console.log('Grammar file newer than parser file, re-compiling') - compile() - } + fs.stat(grammar.parserOutputPath, (err, parserStat) => { + if (err) { + return callback(err) + } + callback() + if (grammarStat.mtime > parserStat.mtime) { + console.log( + 'Grammar file newer than parser file, re-compiling' + ) + compile(grammar) + } + }) }) - }) - } - ) + } + ) + } } } module.exports = { LezerGrammarCompilerPlugin }