Merge pull request #37028 from overleaf/em-bibtex-stream-parser

[web] add a streaming bibtex parser

GitOrigin-RevId: 5397ba546a3d5b3436a80eab0467244969b280d0

services/web/frontend/js/features/bibtex/bibtex-parser.ts

@@ -12,7 +12,10 @@ import {
 import { parser as lezerParser } from '@/features/source-editor/lezer-bibtex/bibtex.mjs'
 import { BibtexEntry } from '@shared/bibtex/bibtex-entry.mts'
 import { PositionedBibtexEntry } from './positioned-bibtex-entry'
-import { BibtexFieldValue } from '@shared/bibtex/bibtex-field-value.mts'
+import {
+  BibtexFieldValue,
+  BibtexFieldValueBuilder,
+} from '@shared/bibtex/bibtex-field-value.mts'
 
 type GetText = (from: number, to: number) => string
 
@@ -43,14 +46,14 @@ export class BibtexEntryAccumulator {
   private key: string | null = null
   private fields: Map<string, BibtexFieldValue> = new Map()
   private fieldName: string | null = null
-  private fieldValue: BibtexFieldValue = new BibtexFieldValue()
+  private fieldValue = new BibtexFieldValueBuilder()
 
   reset() {
     this.type = null
     this.key = null
     this.fields = new Map()
     this.fieldName = null
-    this.fieldValue = new BibtexFieldValue()
+    this.fieldValue = new BibtexFieldValueBuilder()
   }
 
   /**
@@ -75,17 +78,15 @@ export class BibtexEntryAccumulator {
     // not export a term for it. Multi-line indentation is collapsed.
     if (type.name === 'StringContents') {
       const s = getText(node.from, node.to).replaceAll(/[\n\r]\s*/g, ' ')
-      this.fieldValue = this.fieldValue.addString(s)
+      this.fieldValue.addString(s)
       return false
     }
     if (type.is(NumberLiteral)) {
-      this.fieldValue = this.fieldValue.addNumber(getText(node.from, node.to))
+      this.fieldValue.addNumber(getText(node.from, node.to))
       return false
     }
     if (type.is(StringName)) {
-      this.fieldValue = this.fieldValue.addNamedString(
-        getText(node.from, node.to)
-      )
+      this.fieldValue.addNamedString(getText(node.from, node.to))
       return false
     }
     return true
@@ -94,10 +95,10 @@ export class BibtexEntryAccumulator {
   leave(node: SyntaxNodeRef) {
     if (node.type.is(Field)) {
       if (this.fieldName != null) {
-        this.fields.set(this.fieldName, this.fieldValue)
+        this.fields.set(this.fieldName, this.fieldValue.build())
       }
       this.fieldName = null
-      this.fieldValue = new BibtexFieldValue()
+      this.fieldValue = new BibtexFieldValueBuilder()
     }
   }
 

services/web/shared/bibtex/bibtex-field-value.mts

@@ -94,12 +94,6 @@ export class BibtexFieldValue {
     return new BibtexFieldValue([...this.parts, new NamedString(name)])
   }
 
-  private invalidateCache() {
-    this.bibtexString = null
-    this.editableString = null
-    this.displayString = null
-  }
-
   toString() {
     if (this.bibtexString == null) {
       if (this.parts.length === 0) {
@@ -183,7 +177,7 @@ export class BibtexFieldValue {
    *   {This is from 1995}
    */
   static fromEditableString(editable: string) {
-    let bibtexValue = new BibtexFieldValue()
+    const builder = new BibtexFieldValueBuilder()
     let currentString = ''
     let index = 0
 
@@ -209,13 +203,11 @@ export class BibtexFieldValue {
       )
       if (identifierEnd !== null && editable[identifierEnd] === '#') {
         if (currentString !== '') {
-          bibtexValue = bibtexValue.addString(currentString)
+          builder.addString(currentString)
           currentString = ''
         }
 
-        bibtexValue = bibtexValue.addNamedString(
-          editable.slice(nextIndex, identifierEnd)
-        )
+        builder.addNamedString(editable.slice(nextIndex, identifierEnd))
         index = identifierEnd + 1
         continue
       }
@@ -225,10 +217,41 @@ export class BibtexFieldValue {
     }
 
     if (currentString !== '') {
-      bibtexValue = bibtexValue.addString(currentString)
+      builder.addString(currentString)
     }
 
-    return bibtexValue
+    return builder.build()
+  }
+}
+
+/**
+ * Accumulates the parts of a field value in place, for a parser that reads them
+ * one at a time. BibtexFieldValue's own add* methods copy the parts array, so
+ * building a long "#"-concatenated value through them is quadratic.
+ */
+export class BibtexFieldValueBuilder {
+  private readonly parts: Part[] = []
+
+  addString(value: string) {
+    this.parts.push(new StringLiteral(value))
+  }
+
+  addNumber(value: string) {
+    this.parts.push(new NumberLiteral(value))
+  }
+
+  addNamedString(name: string) {
+    this.parts.push(new NamedString(name))
+  }
+
+  /** Whether any part has been added, so "{}" is not an empty builder. */
+  isEmpty(): boolean {
+    return this.parts.length === 0
+  }
+
+  /** The value built so far, unaffected by later additions to the builder. */
+  build(): BibtexFieldValue {
+    return new BibtexFieldValue([...this.parts])
   }
 }
 

services/web/shared/bibtex/parser/bibtex-char-codes.mts

@@ -0,0 +1,49 @@
+import { isBibtexIdentifierContinuationChar } from '../bibtex-identifier.mts'
+
+export const TAB = 0x09
+export const LINE_FEED = 0x0a
+export const CARRIAGE_RETURN = 0x0d
+export const SPACE = 0x20
+export const QUOTE = 0x22
+export const HASH = 0x23
+export const PERCENT = 0x25
+export const OPEN_PAREN = 0x28
+export const CLOSE_PAREN = 0x29
+export const COMMA = 0x2c
+export const EQUALS = 0x3d
+export const AT = 0x40
+export const OPEN_BRACE = 0x7b
+export const CLOSE_BRACE = 0x7d
+
+// Derived from the shared predicate rather than restated, so the character-code
+// variants below cannot drift from it.
+const ASCII_ALLOWED = Uint8Array.from({ length: 0x80 }, (_, code) =>
+  isBibtexIdentifierContinuationChar(String.fromCharCode(code)) ? 1 : 0
+)
+
+/**
+ * Character-code variant of isBibtexIdentifierContinuationChar. Every
+ * forbidden character is ASCII, so anything above is allowed, including the
+ * surrogate halves of an astral character.
+ */
+export function isBibtexIdentifierCharCode(code: number): boolean {
+  return code > 0x7f || ASCII_ALLOWED[code] === 1
+}
+
+/** Character-code variant of isBibtexIdentifierStartChar. */
+export function isBibtexIdentifierStartCharCode(code: number): boolean {
+  return isBibtexIdentifierCharCode(code) && !isDigitCode(code)
+}
+
+export function isDigitCode(code: number): boolean {
+  return code >= 0x30 && code <= 0x39
+}
+
+export function isWhitespaceCode(code: number): boolean {
+  return (
+    code === SPACE ||
+    code === LINE_FEED ||
+    code === TAB ||
+    code === CARRIAGE_RETURN
+  )
+}

services/web/shared/bibtex/parser/bibtex-stream-parser.mts

@@ -0,0 +1,792 @@
+import OError from '@overleaf/o-error'
+import { BibtexEntry } from '../bibtex-entry.mts'
+import {
+  BibtexFieldValue,
+  BibtexFieldValueBuilder,
+} from '../bibtex-field-value.mts'
+import {
+  AT,
+  CARRIAGE_RETURN,
+  CLOSE_BRACE,
+  CLOSE_PAREN,
+  COMMA,
+  EQUALS,
+  HASH,
+  LINE_FEED,
+  OPEN_BRACE,
+  OPEN_PAREN,
+  PERCENT,
+  QUOTE,
+  isBibtexIdentifierCharCode,
+  isBibtexIdentifierStartCharCode,
+  isDigitCode,
+  isWhitespaceCode,
+} from './bibtex-char-codes.mts'
+
+export type BibtexEntryItem = {
+  kind: 'entry'
+  entry: BibtexEntry
+}
+
+/** A `@string` definition, which the parser records rather than expands. */
+export type BibtexNamedStringItem = {
+  kind: 'namedString'
+  /**
+   * Kept as written, as are the references to it. Names are matched
+   * case-insensitively, so a consumer resolving references has to fold case
+   * when it builds its lookup table — but lowercasing here would rewrite the
+   * author's own spelling of the name whenever an entry is read and written
+   * back, which entry types and field names do not have to worry about.
+   *
+   * A definition also takes effect only from where it appears, replacing any
+   * earlier one from that point on, so resolution has to follow item order.
+   */
+  name: string
+  value: BibtexFieldValue
+}
+
+/** A `%` comment or a `@comment` body, without its delimiters. */
+export type BibtexCommentItem = {
+  kind: 'comment'
+  text: string
+}
+
+/** Free-form text between entries, which BibTeX ignores. */
+export type BibtexJunkItem = {
+  kind: 'junk'
+  text: string
+}
+
+export type BibtexItem =
+  | BibtexEntryItem
+  | BibtexNamedStringItem
+  | BibtexCommentItem
+  | BibtexJunkItem
+
+/*
+ * Parser states. Those up to and including AFTER_VALUE sit between entry
+ * tokens, where whitespace and %-comments are skipped; the main loop tests for
+ * them with a single comparison, so their order matters. The IN_* states are
+ * mid-token, and are the only ones a chunk boundary can fall inside.
+ */
+const TOP = 0
+const EXPECT_TYPE = 1
+const EXPECT_BODY_OPEN = 2
+const EXPECT_KEY = 3
+const AFTER_KEY = 4
+const EXPECT_FIELD_NAME = 5
+const EXPECT_EQUALS = 6
+const EXPECT_VALUE = 7
+const AFTER_VALUE = 8
+const LAST_INTER_TOKEN_STATE = AFTER_VALUE
+const IN_TYPE = 9
+const IN_KEY = 10
+const IN_FIELD_NAME = 11
+const IN_NUMBER = 12
+const IN_NAME = 13
+const IN_LITERAL = 14
+const IN_COMMENT_BODY = 15
+const IN_JUNK_WORD = 16
+const IN_JUNK_SPACE = 17
+const IN_LINE_COMMENT = 18
+
+/** What an entry's body holds, selected by its type. */
+const REGULAR = 0 // fields, introduced by a citation key
+const NAMED_STRING = 1 // @string: named string definitions
+const PREAMBLE = 2 // @preamble: a single value
+const COMMENT = 3 // @comment: a literal
+
+/** Mirrors specializeEntryType in the editor's bibtex grammar. */
+function bodyKindOf(lowercasedType: string): number {
+  switch (lowercasedType) {
+    case 'string':
+      return NAMED_STRING
+    case 'preamble':
+      return PREAMBLE
+    case 'comment':
+      return COMMENT
+    default:
+      return REGULAR
+  }
+}
+
+/**
+ * Cap on the input one item may span. Streaming only keeps memory down while
+ * items keep completing: an unterminated literal, a single enormous word of
+ * junk, or an entry with endless fields would otherwise buffer the whole rest of
+ * the input. The limit is far above any real entry, comment or run of junk, so
+ * reaching it means the input is not a .bib file.
+ */
+const MAX_ITEM_LENGTH = 1 << 20
+
+/**
+ * Thrown when one item spans more than MAX_ITEM_LENGTH characters of input. It
+ * leaves the parser part-way through that item, so the parser has to be
+ * discarded rather than written to again.
+ */
+export class BibtexItemTooLargeError extends OError {}
+
+/**
+ * Chunk-oblivious BibTeX parser, built to index .bib files too large to hold a
+ * syntax tree for. It follows the same grammar as the editor's lezer parser
+ * (frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar) and reads
+ * every entry the same way, so a reader sees the same references the editor
+ * shows them. The differential test in the library module pins that.
+ *
+ * Items come out in the order they occur, with one exception: a comment written
+ * inside an entry is reported before the entry that contains it, because the
+ * entry is only complete at its closing delimiter.
+ *
+ * Entries sharing a citation key are all reported, leaving it to the caller to
+ * decide which one wins. `@preamble` bodies are the only content dropped.
+ *
+ * An item spanning more than MAX_ITEM_LENGTH characters of input throws
+ * BibtexItemTooLargeError, so that no input can buffer without bound.
+ */
+export class BibtexStreamParser {
+  private state = TOP
+  private commentReturnState = TOP
+
+  /**
+   * Token text so far, which a chunk boundary may have cut in half. Only one
+   * token is ever in flight: every state that can start a comment or a run of
+   * junk is reached with the buffer already emptied.
+   */
+  private token = ''
+
+  /** Whitespace inside a run of junk, held back until a word follows it. */
+  private pendingSpace = ''
+
+  /**
+   * Input consumed towards the item being read, against MAX_ITEM_LENGTH.
+   * Everything the parser holds on to between chunks — the token in flight, the
+   * entry's type, key and fields, the parts of the value being built — was read
+   * from this stretch of input, so bounding it bounds all of them.
+   */
+  private consumed = 0
+
+  private bodyKind = REGULAR
+
+  /** Closing delimiter the entry body's opener calls for. */
+  private bodyTerminator = CLOSE_BRACE
+
+  private entryType: string | null = null
+  private entryKey: string | null = null
+  private fields = new Map<string, BibtexFieldValue>()
+  private fieldName: string | null = null
+
+  /**
+   * The parts of the value being read. An empty builder separates "foo" and
+   * "foo =", which define nothing — biber rejects both as syntax errors — from
+   * "foo = {}", which defines the empty string and so adds a part.
+   */
+  private fieldValue = new BibtexFieldValueBuilder()
+
+  /** Closing delimiter of the literal being scanned, and its brace nesting. */
+  private literalTerminator = CLOSE_BRACE
+  private literalDepth = 0
+
+  private items: BibtexItem[] = []
+
+  /** Parses a chunk, returning the items it completed. */
+  write(chunk: string): BibtexItem[] {
+    let i = 0
+    while (i < chunk.length) {
+      const state = this.state
+
+      if (state <= LAST_INTER_TOKEN_STATE) {
+        const code = chunk.charCodeAt(i)
+        if (isWhitespaceCode(code)) {
+          // Whitespace between tokens is dropped, so it costs no memory and
+          // goes uncounted: a file can hold as much of it as it likes.
+          i += 1
+          continue
+        }
+        if (code === PERCENT) {
+          this.commentReturnState = state
+          this.state = IN_LINE_COMMENT
+          i += 1
+          continue
+        }
+      }
+
+      const start = i
+
+      switch (state) {
+        case TOP:
+          i = this.parseTop(chunk, i)
+          break
+        case EXPECT_TYPE:
+          i = this.parseExpectType(chunk, i)
+          break
+        case EXPECT_BODY_OPEN:
+          i = this.parseExpectBodyOpen(chunk, i)
+          break
+        case EXPECT_KEY:
+          i = this.parseExpectKey(chunk, i)
+          break
+        case AFTER_KEY:
+          i = this.parseAfterKey(chunk, i)
+          break
+        case EXPECT_FIELD_NAME:
+          i = this.parseExpectFieldName(chunk, i)
+          break
+        case EXPECT_EQUALS:
+          i = this.parseExpectEquals(chunk, i)
+          break
+        case EXPECT_VALUE:
+          i = this.parseExpectValue(chunk, i)
+          break
+        case AFTER_VALUE:
+          i = this.parseAfterValue(chunk, i)
+          break
+        case IN_TYPE:
+          i = this.scanType(chunk, i)
+          break
+        case IN_KEY:
+          i = this.scanKey(chunk, i)
+          break
+        case IN_FIELD_NAME:
+          i = this.scanFieldName(chunk, i)
+          break
+        case IN_NUMBER:
+          i = this.scanNumber(chunk, i)
+          break
+        case IN_NAME:
+          i = this.scanName(chunk, i)
+          break
+        case IN_LITERAL:
+          i = this.scanValueLiteral(chunk, i)
+          break
+        case IN_COMMENT_BODY:
+          i = this.scanCommentBody(chunk, i)
+          break
+        case IN_JUNK_WORD:
+          i = this.scanJunkWord(chunk, i)
+          break
+        case IN_JUNK_SPACE:
+          i = this.scanJunkSpace(chunk, i)
+          break
+        case IN_LINE_COMMENT:
+          i = this.scanLineComment(chunk, i)
+          break
+      }
+
+      this.consume(i - start)
+    }
+    return this.takeItems()
+  }
+
+  /**
+   * Signals the end of the input, returning any final items. An entry left
+   * unterminated still yields what was parsed of it, as it does in the editor.
+   */
+  end(): BibtexItem[] {
+    switch (this.state) {
+      case IN_TYPE:
+        this.finishType()
+        break
+      case IN_KEY:
+        this.finishKey()
+        break
+      case IN_FIELD_NAME:
+        this.finishFieldName()
+        break
+      case IN_NUMBER:
+        this.finishNumber()
+        break
+      case IN_NAME:
+        this.finishName()
+        break
+      case IN_LITERAL:
+        this.finishLiteral()
+        break
+      case IN_COMMENT_BODY:
+        this.emitComment()
+        break
+      case IN_JUNK_WORD:
+      case IN_JUNK_SPACE:
+        this.emitJunk()
+        break
+      case IN_LINE_COMMENT:
+        this.emitComment()
+        break
+    }
+    this.endEntry()
+    this.state = TOP
+    this.consumed = 0
+    return this.takeItems()
+  }
+
+  private parseTop(chunk: string, i: number): number {
+    if (chunk.charCodeAt(i) === AT) {
+      this.beginEntry()
+      this.state = EXPECT_TYPE
+      return i + 1
+    }
+    this.state = IN_JUNK_WORD
+    return i
+  }
+
+  /**
+   * Scans a word of free-form comment text. Where the word ends decides how the
+   * next "%" reads: it only opens a comment at the start of a word, so
+   * "Junk% @article{a}" holds an entry while "Junk % @article{a}" has it
+   * commented out.
+   */
+  private scanJunkWord(chunk: string, i: number): number {
+    const start = i
+    while (i < chunk.length) {
+      const code = chunk.charCodeAt(i)
+      if (code === AT) {
+        this.token += chunk.slice(start, i)
+        this.emitJunk()
+        this.state = TOP
+        return i
+      }
+      if (isWhitespaceCode(code)) {
+        this.token += chunk.slice(start, i)
+        this.state = IN_JUNK_SPACE
+        return i
+      }
+      i += 1
+    }
+    this.token += chunk.slice(start, i)
+    return i
+  }
+
+  private scanJunkSpace(chunk: string, i: number): number {
+    const start = i
+    while (i < chunk.length && isWhitespaceCode(chunk.charCodeAt(i))) {
+      i += 1
+    }
+    this.pendingSpace += chunk.slice(start, i)
+    if (i === chunk.length) {
+      return i
+    }
+
+    const code = chunk.charCodeAt(i)
+    if (code === AT || code === PERCENT) {
+      this.emitJunk()
+      this.state = TOP
+      return i
+    }
+    this.token += this.pendingSpace
+    this.pendingSpace = ''
+    this.state = IN_JUNK_WORD
+    return i
+  }
+
+  private scanLineComment(chunk: string, i: number): number {
+    const start = i
+    while (i < chunk.length) {
+      const code = chunk.charCodeAt(i)
+      if (code === LINE_FEED || code === CARRIAGE_RETURN) {
+        break
+      }
+      i += 1
+    }
+    this.token += chunk.slice(start, i)
+    if (i < chunk.length) {
+      this.emitComment()
+      this.state = this.commentReturnState
+    }
+    return i
+  }
+
+  private parseExpectType(chunk: string, i: number): number {
+    if (isBibtexIdentifierStartCharCode(chunk.charCodeAt(i))) {
+      this.state = IN_TYPE
+      return i
+    }
+    return this.recover(i)
+  }
+
+  private scanType(chunk: string, i: number): number {
+    i = this.scanIdentifier(chunk, i)
+    if (i < chunk.length) {
+      this.finishType()
+    }
+    return i
+  }
+
+  private finishType() {
+    const type = this.takeToken().toLowerCase()
+    this.entryType = type
+    this.bodyKind = bodyKindOf(type)
+    this.state = EXPECT_BODY_OPEN
+  }
+
+  private parseExpectBodyOpen(chunk: string, i: number): number {
+    const code = chunk.charCodeAt(i)
+    if (code !== OPEN_BRACE && code !== OPEN_PAREN) {
+      return this.recover(i)
+    }
+    this.bodyTerminator = code === OPEN_BRACE ? CLOSE_BRACE : CLOSE_PAREN
+    switch (this.bodyKind) {
+      case COMMENT:
+        // A @comment body is a literal rather than a list of values, so it
+        // swallows any entry written inside it.
+        this.beginLiteral(this.bodyTerminator)
+        this.state = IN_COMMENT_BODY
+        break
+      case PREAMBLE:
+        this.state = EXPECT_VALUE
+        break
+      case NAMED_STRING:
+        this.state = EXPECT_FIELD_NAME
+        break
+      default:
+        this.state = EXPECT_KEY
+    }
+    return i + 1
+  }
+
+  private parseExpectKey(chunk: string, i: number): number {
+    if (isBibtexIdentifierCharCode(chunk.charCodeAt(i))) {
+      this.state = IN_KEY
+      return i
+    }
+    return this.recover(i)
+  }
+
+  private scanKey(chunk: string, i: number): number {
+    i = this.scanIdentifier(chunk, i)
+    if (i < chunk.length) {
+      this.finishKey()
+    }
+    return i
+  }
+
+  private finishKey() {
+    // Citation keys keep their case, unlike types and field names.
+    this.entryKey = this.takeToken()
+    this.state = AFTER_KEY
+  }
+
+  private parseAfterKey(chunk: string, i: number): number {
+    const code = chunk.charCodeAt(i)
+    if (code === COMMA) {
+      this.state = EXPECT_FIELD_NAME
+      return i + 1
+    }
+    if (code === this.bodyTerminator) {
+      return this.closeBody(i)
+    }
+    return this.recover(i)
+  }
+
+  private parseExpectFieldName(chunk: string, i: number): number {
+    const code = chunk.charCodeAt(i)
+    if (code === this.bodyTerminator) {
+      return this.closeBody(i)
+    }
+    if (isBibtexIdentifierStartCharCode(code)) {
+      this.state = IN_FIELD_NAME
+      return i
+    }
+    return this.recover(i)
+  }
+
+  private scanFieldName(chunk: string, i: number): number {
+    i = this.scanIdentifier(chunk, i)
+    if (i < chunk.length) {
+      this.finishFieldName()
+    }
+    return i
+  }
+
+  private finishFieldName() {
+    this.fieldName = this.takeToken()
+    this.state = EXPECT_EQUALS
+  }
+
+  private parseExpectEquals(chunk: string, i: number): number {
+    const code = chunk.charCodeAt(i)
+    if (code === EQUALS) {
+      this.state = EXPECT_VALUE
+      return i + 1
+    }
+    if (code === this.bodyTerminator) {
+      return this.closeBody(i)
+    }
+    return this.recover(i)
+  }
+
+  private parseExpectValue(chunk: string, i: number): number {
+    const code = chunk.charCodeAt(i)
+    if (code === OPEN_BRACE || code === QUOTE) {
+      this.beginLiteral(code === OPEN_BRACE ? CLOSE_BRACE : QUOTE)
+      this.state = IN_LITERAL
+      return i + 1
+    }
+    if (isDigitCode(code)) {
+      this.state = IN_NUMBER
+      return i
+    }
+    if (isBibtexIdentifierStartCharCode(code)) {
+      this.state = IN_NAME
+      return i
+    }
+    // A missing value contributes no part, so "title=," and "title={x} #}"
+    // behave as if the value had already ended here.
+    this.state = AFTER_VALUE
+    return i
+  }
+
+  private scanNumber(chunk: string, i: number): number {
+    const start = i
+    while (i < chunk.length && isDigitCode(chunk.charCodeAt(i))) {
+      i += 1
+    }
+    this.token += chunk.slice(start, i)
+    if (i < chunk.length) {
+      this.finishNumber()
+    }
+    return i
+  }
+
+  private finishNumber() {
+    this.fieldValue.addNumber(this.takeToken())
+    this.state = AFTER_VALUE
+  }
+
+  private scanName(chunk: string, i: number): number {
+    i = this.scanIdentifier(chunk, i)
+    if (i < chunk.length) {
+      this.finishName()
+    }
+    return i
+  }
+
+  private finishName() {
+    // A reference to a named string is recorded, not expanded.
+    this.fieldValue.addNamedString(this.takeToken())
+    this.state = AFTER_VALUE
+  }
+
+  private scanValueLiteral(chunk: string, i: number): number {
+    i = this.scanLiteral(chunk, i)
+    if (i < chunk.length) {
+      this.finishLiteral()
+      return i + 1
+    }
+    return i
+  }
+
+  private finishLiteral() {
+    // A line break and the indentation following it collapse to one space.
+    this.fieldValue.addString(this.takeToken().replaceAll(/[\n\r]\s*/g, ' '))
+    this.state = AFTER_VALUE
+  }
+
+  private scanCommentBody(chunk: string, i: number): number {
+    i = this.scanLiteral(chunk, i)
+    if (i < chunk.length) {
+      this.emitComment()
+      this.endEntry()
+      this.state = TOP
+      return i + 1
+    }
+    return i
+  }
+
+  private parseAfterValue(chunk: string, i: number): number {
+    const code = chunk.charCodeAt(i)
+    if (code === HASH) {
+      this.state = EXPECT_VALUE
+      return i + 1
+    }
+    if (code === COMMA) {
+      this.commitField()
+      this.state = EXPECT_FIELD_NAME
+      return i + 1
+    }
+    if (code === this.bodyTerminator) {
+      return this.closeBody(i)
+    }
+    return this.recover(i)
+  }
+
+  /** Appends identifier characters to the token buffer. */
+  private scanIdentifier(chunk: string, i: number): number {
+    const start = i
+    while (
+      i < chunk.length &&
+      isBibtexIdentifierCharCode(chunk.charCodeAt(i))
+    ) {
+      i += 1
+    }
+    this.token += chunk.slice(start, i)
+    return i
+  }
+
+  private beginLiteral(terminator: number) {
+    this.literalTerminator = terminator
+    this.literalDepth = 0
+  }
+
+  /**
+   * Appends literal text to the token buffer, stopping at its closing
+   * delimiter. Only braces nest, and there are no backslash escapes: every
+   * other delimiter is content until the nesting is back to zero.
+   */
+  private scanLiteral(chunk: string, i: number): number {
+    const start = i
+    while (i < chunk.length) {
+      const code = chunk.charCodeAt(i)
+      if (code === OPEN_BRACE) {
+        this.literalDepth += 1
+      } else if (code === CLOSE_BRACE && this.literalDepth > 0) {
+        this.literalDepth -= 1
+      } else if (this.literalDepth === 0 && code === this.literalTerminator) {
+        break
+      }
+      i += 1
+    }
+    this.token += chunk.slice(start, i)
+    return i
+  }
+
+  /**
+   * Counts the input a step of the main loop read towards the item in progress.
+   * Landing back at TOP means the item is complete and the parser is holding
+   * nothing, so the count starts over from there.
+   */
+  private consume(length: number) {
+    if (this.state === TOP) {
+      this.consumed = 0
+      return
+    }
+    this.consumed += length
+    if (this.consumed > MAX_ITEM_LENGTH) {
+      throw new BibtexItemTooLargeError('bibtex item is too large', {
+        limit: MAX_ITEM_LENGTH,
+      })
+    }
+  }
+
+  private takeToken(): string {
+    const token = this.token
+    this.token = ''
+    return token
+  }
+
+  private emitJunk() {
+    const text = this.takeToken()
+    this.pendingSpace = ''
+    if (text !== '') {
+      this.items.push({ kind: 'junk', text })
+    }
+  }
+
+  private emitComment() {
+    const text = this.takeToken()
+    // A comment can sit inside an entry, whose fields are still held, so only
+    // the comment's own share of the input is given back.
+    this.consumed -= text.length
+    this.items.push({ kind: 'comment', text })
+  }
+
+  private beginEntry() {
+    this.bodyKind = REGULAR
+    this.entryType = null
+    this.entryKey = null
+    this.fields = new Map()
+    this.fieldName = null
+    this.fieldValue = new BibtexFieldValueBuilder()
+    this.token = ''
+  }
+
+  private commitField() {
+    if (this.fieldName != null) {
+      if (this.bodyKind === NAMED_STRING) {
+        // "@string{foo}" and "@string{foo =}" define nothing, so reporting them
+        // would let either shadow an earlier valid definition of foo.
+        if (!this.fieldValue.isEmpty()) {
+          this.items.push({
+            kind: 'namedString',
+            name: this.fieldName,
+            value: this.fieldValue.build(),
+          })
+        }
+      } else {
+        // A field with no value is still committed, to match the editor.
+        this.fields.set(this.fieldName.toLowerCase(), this.fieldValue.build())
+      }
+    }
+    this.fieldName = null
+    this.fieldValue = new BibtexFieldValueBuilder()
+  }
+
+  private closeBody(i: number): number {
+    this.endEntry()
+    this.state = TOP
+    return i + 1
+  }
+
+  /**
+   * Ends the entry where it stands after an unexpected character, keeping the
+   * fields parsed so far. Leaving the character for the top level to look at
+   * again is what lets an unclosed entry hand the next "@" to a fresh one.
+   */
+  private recover(i: number): number {
+    this.endEntry()
+    this.state = TOP
+    return i
+  }
+
+  /**
+   * Completes the entry in progress, reporting it if it has both a type and a
+   * citation key.
+   */
+  private endEntry() {
+    this.commitField()
+    if (
+      this.bodyKind === REGULAR &&
+      this.entryType != null &&
+      this.entryKey != null
+    ) {
+      this.items.push({
+        kind: 'entry',
+        entry: new BibtexEntry({
+          type: this.entryType,
+          key: this.entryKey,
+          fields: this.fields,
+        }),
+      })
+    }
+    this.entryType = null
+    this.entryKey = null
+    this.fields = new Map()
+    this.token = ''
+  }
+
+  private takeItems(): BibtexItem[] {
+    const items = this.items
+    this.items = []
+    return items
+  }
+}
+
+/** Parses a whole .bib file held in memory. */
+export function parseBibtexItems(text: string): BibtexItem[] {
+  const parser = new BibtexStreamParser()
+  const items = parser.write(text)
+  return items.concat(parser.end())
+}
+
+/**
+ * Parses a .bib file arriving in chunks. The caller is responsible for decoding
+ * bytes with a stateful decoder, so a multi-byte character split across chunks
+ * is not corrupted.
+ */
+export async function* streamBibtexItems(
+  chunks: AsyncIterable<string>
+): AsyncGenerator<BibtexItem> {
+  const parser = new BibtexStreamParser()
+  for await (const chunk of chunks) {
+    yield* parser.write(chunk)
+  }
+  yield* parser.end()
+}