Merge pull request #37005 from overleaf/em-bibtex-grammar-fixes

[web] match biber in three bibtex parsing edge cases

GitOrigin-RevId: da7e2c0b9f137e7debfe26f09c531a61a135e40e

services/web/frontend/js/features/bibtex/bibtex-parser.ts

@@ -6,7 +6,6 @@ import {
   CitationKey,
   Field,
   FieldName,
-  StringLiteral,
   NumberLiteral,
   StringName,
 } from '@/features/source-editor/lezer-bibtex/bibtex.terms.mjs'
@@ -72,10 +71,10 @@ export class BibtexEntryAccumulator {
       this.fieldName = getText(node.from, node.to).toLowerCase()
       return false
     }
-    if (type.is(StringLiteral)) {
-      // Strip surrounding quotes/braces and collapse multi-line indentation.
-      let s = getText(node.from + 1, node.to - 1)
-      s = s.replaceAll(/[\n\r]\s*/g, ' ')
+    // StringContents spans a literal's text without its delimiters; lezer does
+    // not export a term for it. Multi-line indentation is collapsed.
+    if (type.name === 'StringContents') {
+      const s = getText(node.from, node.to).replaceAll(/[\n\r]\s*/g, ' ')
       this.fieldValue = this.fieldValue.addString(s)
       return false
     }

services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar

@@ -1,24 +1,34 @@
 @tokens {
   whiteSpace { $[\t\n\r ]+ }
 
-  // Anything outside of an entry at the top level is a BibTeX free-form comment
-  Junk { ![@ \t\n\r%] ![@]* }
+  // Anything outside of an entry at the top level is a BibTeX free-form comment.
+  // A word may contain a "%" but may not start with one, so a "%" at a token
+  // boundary still opens a comment: btparse parses the entry in
+  // "Junk% @article{a}" but treats "Junk % @article{a}" as a comment.
+  junkWord { ![@ \t\n\r%] ![@ \t\n\r]* }
+  Junk { junkWord (whiteSpace junkWord)* }
 
   // BibLaTeX supports comments starting with a % and extending to the end of
   // the line
   Comment { "%" ![\n\r]* }
 
-  // See https://metacpan.org/release/AMBS/Text-BibTeX-0.91/view/btparse/doc/bt_language.pod
-  identifierSymbol { $[!$&*+./:;<>?^`_|] | "[" | "]" | "-" }
-  identifier {
-      (@asciiLetter | identifierSymbol)
-      (@asciiLetter | @digit | identifierSymbol)*
-  }
+  // btparse derives a name by "removing the magic 10 from printable ASCII, then
+  // further excluding @, \ and ~", the magic 10 being " # % ' ( ) , = { } and
+  // space. Stating that by exclusion rather than by listing the allowed
+  // punctuation also admits non-ASCII: btparse's own lexer is ASCII-only here,
+  // but biber accepts UTF-8 throughout, including in citation keys.
+  // See https://metacpan.org/dist/Text-BibTeX/view/btparse/doc/bt_language.pod
+  identifierChar { ![\u{0}-\u{20}\u{7f}"#%'(),=@\\{}~] }
+
+  // Identifiers cannot start with a digit, so that a value like 1995 is a
+  // NumberLiteral rather than a macro name. Citation keys can.
+  identifierStart { ![\u{0}-\u{20}\u{7f}"#%'(),=@\\{}~0-9] }
+
+  identifier { identifierStart identifierChar* }
 
   EntryType { identifier }
 
-  // Citation keys can start with a digit
-  CitationKey { (@asciiLetter | @digit | identifierSymbol)+ }
+  CitationKey { identifierChar+ }
 
   NumberLiteral { @digit+ }
 

services/web/shared/bibtex/bibtex-field-value.mts

@@ -1,7 +1,9 @@
 import { latexToUnicode } from './latex-to-unicode.mts'
 import { purifySpecialChars } from './bibtex-purify.mts'
-
-const IDENTIFIER_SYMBOLS = '!$&*+./:;<>?^`_|[]-'
+import {
+  isBibtexIdentifierContinuationChar,
+  isBibtexIdentifierStartChar,
+} from './bibtex-identifier.mts'
 
 class StringLiteral {
   private value: string
@@ -165,10 +167,8 @@ export class BibtexFieldValue {
    * 3. Any other '#' is treated as a literal '#'.
    * 4. Literal runs become braced strings and parts are joined with ' # '.
    *
-   * Identifier rule:
-   * - First char: letter or one of ! $ & * + . / : ; < > ? ^ ` _ | [ ] -
-   * - Following chars: same set plus digits.
-   * - Cannot start with a digit.
+   * Abbreviation names use the identifier character class from
+   * bibtex-identifier.mts, which cannot start with a digit.
    *
    * Example:
    * Editable:
@@ -257,22 +257,6 @@ function findBibtexIdentifierEndExclusive(
   return index
 }
 
-function isBibtexIdentifierStartChar(char: string): boolean {
-  return isAsciiLetter(char) || IDENTIFIER_SYMBOLS.includes(char)
-}
-
-function isBibtexIdentifierContinuationChar(char: string): boolean {
-  return isBibtexIdentifierStartChar(char) || isDigit(char)
-}
-
-function isAsciiLetter(char: string): boolean {
-  return /^[A-Za-z]$/.test(char)
-}
-
-function isDigit(char: string): boolean {
-  return /^[0-9]$/.test(char)
-}
-
 export const __test = {
   findBibtexIdentifierEndExclusive,
 }

services/web/shared/bibtex/bibtex-identifier.mts

@@ -0,0 +1,39 @@
+/**
+ * The BibTeX identifier character class, shared by entry types, citation keys,
+ * field names and abbreviation names.
+ *
+ * btparse derives it by "removing the magic 10 from printable ASCII, then
+ * further excluding @, \ and ~", the magic 10 being " # % ' ( ) , = { } and
+ * space. Stated by exclusion it also admits non-ASCII, which biber accepts in
+ * all four positions.
+ *
+ * Mirrors the identifierChar and identifierStart rules in
+ * frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar, so keep the
+ * two in step.
+ *
+ * See https://metacpan.org/dist/Text-BibTeX/view/btparse/doc/bt_language.pod
+ */
+
+// Control characters and DEL are excluded along with the punctuation, so a key
+// cannot carry an invisible ASCII character.
+const FORBIDDEN = '\\u0000-\\u0020\\u007f"#%\'(),=@\\\\{}~'
+
+/** Citation keys, which unlike other identifiers may start with a digit. */
+export const BIBTEX_CITATION_KEY_REGEX = new RegExp(`^[^${FORBIDDEN}]+$`)
+
+/** Entry types, field names and abbreviation names. */
+export const BIBTEX_IDENTIFIER_REGEX = new RegExp(
+  `^[^${FORBIDDEN}0-9][^${FORBIDDEN}]*$`
+)
+
+const FORBIDDEN_CHAR = new RegExp(`[${FORBIDDEN}]`)
+
+export function isBibtexIdentifierStartChar(char: string): boolean {
+  return (
+    isBibtexIdentifierContinuationChar(char) && !(char >= '0' && char <= '9')
+  )
+}
+
+export function isBibtexIdentifierContinuationChar(char: string): boolean {
+  return char !== '' && !FORBIDDEN_CHAR.test(char)
+}