From 0f22a44d1aa15a3f1c5782f2ce597e1e1f81812f Mon Sep 17 00:00:00 2001 From: Eric Mc Sween <5454374+emcsween@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:27:34 -0400 Subject: [PATCH] Merge pull request #37005 from overleaf/em-bibtex-grammar-fixes [web] match biber in three bibtex parsing edge cases GitOrigin-RevId: da7e2c0b9f137e7debfe26f09c531a61a135e40e --- .../js/features/bibtex/bibtex-parser.ts | 9 ++--- .../source-editor/lezer-bibtex/bibtex.grammar | 30 +++++++++----- .../web/shared/bibtex/bibtex-field-value.mts | 28 +++---------- .../web/shared/bibtex/bibtex-identifier.mts | 39 +++++++++++++++++++ 4 files changed, 69 insertions(+), 37 deletions(-) create mode 100644 services/web/shared/bibtex/bibtex-identifier.mts diff --git a/services/web/frontend/js/features/bibtex/bibtex-parser.ts b/services/web/frontend/js/features/bibtex/bibtex-parser.ts index f4235c9e99..0a279efc73 100644 --- a/services/web/frontend/js/features/bibtex/bibtex-parser.ts +++ b/services/web/frontend/js/features/bibtex/bibtex-parser.ts @@ -6,7 +6,6 @@ import { CitationKey, Field, FieldName, - StringLiteral, NumberLiteral, StringName, } from '@/features/source-editor/lezer-bibtex/bibtex.terms.mjs' @@ -72,10 +71,10 @@ export class BibtexEntryAccumulator { this.fieldName = getText(node.from, node.to).toLowerCase() return false } - if (type.is(StringLiteral)) { - // Strip surrounding quotes/braces and collapse multi-line indentation. - let s = getText(node.from + 1, node.to - 1) - s = s.replaceAll(/[\n\r]\s*/g, ' ') + // StringContents spans a literal's text without its delimiters; lezer does + // not export a term for it. Multi-line indentation is collapsed. + if (type.name === 'StringContents') { + const s = getText(node.from, node.to).replaceAll(/[\n\r]\s*/g, ' ') this.fieldValue = this.fieldValue.addString(s) return false } diff --git a/services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar b/services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar index 595623f6e9..5f9499f307 100644 --- a/services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar +++ b/services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar @@ -1,24 +1,34 @@ @tokens { whiteSpace { $[\t\n\r ]+ } - // Anything outside of an entry at the top level is a BibTeX free-form comment - Junk { ![@ \t\n\r%] ![@]* } + // Anything outside of an entry at the top level is a BibTeX free-form comment. + // A word may contain a "%" but may not start with one, so a "%" at a token + // boundary still opens a comment: btparse parses the entry in + // "Junk% @article{a}" but treats "Junk % @article{a}" as a comment. + junkWord { ![@ \t\n\r%] ![@ \t\n\r]* } + Junk { junkWord (whiteSpace junkWord)* } // BibLaTeX supports comments starting with a % and extending to the end of // the line Comment { "%" ![\n\r]* } - // See https://metacpan.org/release/AMBS/Text-BibTeX-0.91/view/btparse/doc/bt_language.pod - identifierSymbol { $[!$&*+./:;<>?^`_|] | "[" | "]" | "-" } - identifier { - (@asciiLetter | identifierSymbol) - (@asciiLetter | @digit | identifierSymbol)* - } + // btparse derives a name by "removing the magic 10 from printable ASCII, then + // further excluding @, \ and ~", the magic 10 being " # % ' ( ) , = { } and + // space. Stating that by exclusion rather than by listing the allowed + // punctuation also admits non-ASCII: btparse's own lexer is ASCII-only here, + // but biber accepts UTF-8 throughout, including in citation keys. + // See https://metacpan.org/dist/Text-BibTeX/view/btparse/doc/bt_language.pod + identifierChar { ![\u{0}-\u{20}\u{7f}"#%'(),=@\\{}~] } + + // Identifiers cannot start with a digit, so that a value like 1995 is a + // NumberLiteral rather than a macro name. Citation keys can. + identifierStart { ![\u{0}-\u{20}\u{7f}"#%'(),=@\\{}~0-9] } + + identifier { identifierStart identifierChar* } EntryType { identifier } - // Citation keys can start with a digit - CitationKey { (@asciiLetter | @digit | identifierSymbol)+ } + CitationKey { identifierChar+ } NumberLiteral { @digit+ } diff --git a/services/web/shared/bibtex/bibtex-field-value.mts b/services/web/shared/bibtex/bibtex-field-value.mts index 475b90de01..46151fd435 100644 --- a/services/web/shared/bibtex/bibtex-field-value.mts +++ b/services/web/shared/bibtex/bibtex-field-value.mts @@ -1,7 +1,9 @@ import { latexToUnicode } from './latex-to-unicode.mts' import { purifySpecialChars } from './bibtex-purify.mts' - -const IDENTIFIER_SYMBOLS = '!$&*+./:;<>?^`_|[]-' +import { + isBibtexIdentifierContinuationChar, + isBibtexIdentifierStartChar, +} from './bibtex-identifier.mts' class StringLiteral { private value: string @@ -165,10 +167,8 @@ export class BibtexFieldValue { * 3. Any other '#' is treated as a literal '#'. * 4. Literal runs become braced strings and parts are joined with ' # '. * - * Identifier rule: - * - First char: letter or one of ! $ & * + . / : ; < > ? ^ ` _ | [ ] - - * - Following chars: same set plus digits. - * - Cannot start with a digit. + * Abbreviation names use the identifier character class from + * bibtex-identifier.mts, which cannot start with a digit. * * Example: * Editable: @@ -257,22 +257,6 @@ function findBibtexIdentifierEndExclusive( return index } -function isBibtexIdentifierStartChar(char: string): boolean { - return isAsciiLetter(char) || IDENTIFIER_SYMBOLS.includes(char) -} - -function isBibtexIdentifierContinuationChar(char: string): boolean { - return isBibtexIdentifierStartChar(char) || isDigit(char) -} - -function isAsciiLetter(char: string): boolean { - return /^[A-Za-z]$/.test(char) -} - -function isDigit(char: string): boolean { - return /^[0-9]$/.test(char) -} - export const __test = { findBibtexIdentifierEndExclusive, } diff --git a/services/web/shared/bibtex/bibtex-identifier.mts b/services/web/shared/bibtex/bibtex-identifier.mts new file mode 100644 index 0000000000..8813104bfc --- /dev/null +++ b/services/web/shared/bibtex/bibtex-identifier.mts @@ -0,0 +1,39 @@ +/** + * The BibTeX identifier character class, shared by entry types, citation keys, + * field names and abbreviation names. + * + * btparse derives it by "removing the magic 10 from printable ASCII, then + * further excluding @, \ and ~", the magic 10 being " # % ' ( ) , = { } and + * space. Stated by exclusion it also admits non-ASCII, which biber accepts in + * all four positions. + * + * Mirrors the identifierChar and identifierStart rules in + * frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar, so keep the + * two in step. + * + * See https://metacpan.org/dist/Text-BibTeX/view/btparse/doc/bt_language.pod + */ + +// Control characters and DEL are excluded along with the punctuation, so a key +// cannot carry an invisible ASCII character. +const FORBIDDEN = '\\u0000-\\u0020\\u007f"#%\'(),=@\\\\{}~' + +/** Citation keys, which unlike other identifiers may start with a digit. */ +export const BIBTEX_CITATION_KEY_REGEX = new RegExp(`^[^${FORBIDDEN}]+$`) + +/** Entry types, field names and abbreviation names. */ +export const BIBTEX_IDENTIFIER_REGEX = new RegExp( + `^[^${FORBIDDEN}0-9][^${FORBIDDEN}]*$` +) + +const FORBIDDEN_CHAR = new RegExp(`[${FORBIDDEN}]`) + +export function isBibtexIdentifierStartChar(char: string): boolean { + return ( + isBibtexIdentifierContinuationChar(char) && !(char >= '0' && char <= '9') + ) +} + +export function isBibtexIdentifierContinuationChar(char: string): boolean { + return char !== '' && !FORBIDDEN_CHAR.test(char) +}