diff --git a/services/web/frontend/js/features/bibtex/bibtex-parser.ts b/services/web/frontend/js/features/bibtex/bibtex-parser.ts index 0a279efc73..f71d4ba1d6 100644 --- a/services/web/frontend/js/features/bibtex/bibtex-parser.ts +++ b/services/web/frontend/js/features/bibtex/bibtex-parser.ts @@ -12,7 +12,10 @@ import { import { parser as lezerParser } from '@/features/source-editor/lezer-bibtex/bibtex.mjs' import { BibtexEntry } from '@shared/bibtex/bibtex-entry.mts' import { PositionedBibtexEntry } from './positioned-bibtex-entry' -import { BibtexFieldValue } from '@shared/bibtex/bibtex-field-value.mts' +import { + BibtexFieldValue, + BibtexFieldValueBuilder, +} from '@shared/bibtex/bibtex-field-value.mts' type GetText = (from: number, to: number) => string @@ -43,14 +46,14 @@ export class BibtexEntryAccumulator { private key: string | null = null private fields: Map = new Map() private fieldName: string | null = null - private fieldValue: BibtexFieldValue = new BibtexFieldValue() + private fieldValue = new BibtexFieldValueBuilder() reset() { this.type = null this.key = null this.fields = new Map() this.fieldName = null - this.fieldValue = new BibtexFieldValue() + this.fieldValue = new BibtexFieldValueBuilder() } /** @@ -75,17 +78,15 @@ export class BibtexEntryAccumulator { // not export a term for it. Multi-line indentation is collapsed. if (type.name === 'StringContents') { const s = getText(node.from, node.to).replaceAll(/[\n\r]\s*/g, ' ') - this.fieldValue = this.fieldValue.addString(s) + this.fieldValue.addString(s) return false } if (type.is(NumberLiteral)) { - this.fieldValue = this.fieldValue.addNumber(getText(node.from, node.to)) + this.fieldValue.addNumber(getText(node.from, node.to)) return false } if (type.is(StringName)) { - this.fieldValue = this.fieldValue.addNamedString( - getText(node.from, node.to) - ) + this.fieldValue.addNamedString(getText(node.from, node.to)) return false } return true @@ -94,10 +95,10 @@ export class BibtexEntryAccumulator { leave(node: SyntaxNodeRef) { if (node.type.is(Field)) { if (this.fieldName != null) { - this.fields.set(this.fieldName, this.fieldValue) + this.fields.set(this.fieldName, this.fieldValue.build()) } this.fieldName = null - this.fieldValue = new BibtexFieldValue() + this.fieldValue = new BibtexFieldValueBuilder() } } diff --git a/services/web/shared/bibtex/bibtex-field-value.mts b/services/web/shared/bibtex/bibtex-field-value.mts index 46151fd435..652abb0e41 100644 --- a/services/web/shared/bibtex/bibtex-field-value.mts +++ b/services/web/shared/bibtex/bibtex-field-value.mts @@ -94,12 +94,6 @@ export class BibtexFieldValue { return new BibtexFieldValue([...this.parts, new NamedString(name)]) } - private invalidateCache() { - this.bibtexString = null - this.editableString = null - this.displayString = null - } - toString() { if (this.bibtexString == null) { if (this.parts.length === 0) { @@ -183,7 +177,7 @@ export class BibtexFieldValue { * {This is from 1995} */ static fromEditableString(editable: string) { - let bibtexValue = new BibtexFieldValue() + const builder = new BibtexFieldValueBuilder() let currentString = '' let index = 0 @@ -209,13 +203,11 @@ export class BibtexFieldValue { ) if (identifierEnd !== null && editable[identifierEnd] === '#') { if (currentString !== '') { - bibtexValue = bibtexValue.addString(currentString) + builder.addString(currentString) currentString = '' } - bibtexValue = bibtexValue.addNamedString( - editable.slice(nextIndex, identifierEnd) - ) + builder.addNamedString(editable.slice(nextIndex, identifierEnd)) index = identifierEnd + 1 continue } @@ -225,10 +217,41 @@ export class BibtexFieldValue { } if (currentString !== '') { - bibtexValue = bibtexValue.addString(currentString) + builder.addString(currentString) } - return bibtexValue + return builder.build() + } +} + +/** + * Accumulates the parts of a field value in place, for a parser that reads them + * one at a time. BibtexFieldValue's own add* methods copy the parts array, so + * building a long "#"-concatenated value through them is quadratic. + */ +export class BibtexFieldValueBuilder { + private readonly parts: Part[] = [] + + addString(value: string) { + this.parts.push(new StringLiteral(value)) + } + + addNumber(value: string) { + this.parts.push(new NumberLiteral(value)) + } + + addNamedString(name: string) { + this.parts.push(new NamedString(name)) + } + + /** Whether any part has been added, so "{}" is not an empty builder. */ + isEmpty(): boolean { + return this.parts.length === 0 + } + + /** The value built so far, unaffected by later additions to the builder. */ + build(): BibtexFieldValue { + return new BibtexFieldValue([...this.parts]) } } diff --git a/services/web/shared/bibtex/parser/bibtex-char-codes.mts b/services/web/shared/bibtex/parser/bibtex-char-codes.mts new file mode 100644 index 0000000000..c0d756a87b --- /dev/null +++ b/services/web/shared/bibtex/parser/bibtex-char-codes.mts @@ -0,0 +1,49 @@ +import { isBibtexIdentifierContinuationChar } from '../bibtex-identifier.mts' + +export const TAB = 0x09 +export const LINE_FEED = 0x0a +export const CARRIAGE_RETURN = 0x0d +export const SPACE = 0x20 +export const QUOTE = 0x22 +export const HASH = 0x23 +export const PERCENT = 0x25 +export const OPEN_PAREN = 0x28 +export const CLOSE_PAREN = 0x29 +export const COMMA = 0x2c +export const EQUALS = 0x3d +export const AT = 0x40 +export const OPEN_BRACE = 0x7b +export const CLOSE_BRACE = 0x7d + +// Derived from the shared predicate rather than restated, so the character-code +// variants below cannot drift from it. +const ASCII_ALLOWED = Uint8Array.from({ length: 0x80 }, (_, code) => + isBibtexIdentifierContinuationChar(String.fromCharCode(code)) ? 1 : 0 +) + +/** + * Character-code variant of isBibtexIdentifierContinuationChar. Every + * forbidden character is ASCII, so anything above is allowed, including the + * surrogate halves of an astral character. + */ +export function isBibtexIdentifierCharCode(code: number): boolean { + return code > 0x7f || ASCII_ALLOWED[code] === 1 +} + +/** Character-code variant of isBibtexIdentifierStartChar. */ +export function isBibtexIdentifierStartCharCode(code: number): boolean { + return isBibtexIdentifierCharCode(code) && !isDigitCode(code) +} + +export function isDigitCode(code: number): boolean { + return code >= 0x30 && code <= 0x39 +} + +export function isWhitespaceCode(code: number): boolean { + return ( + code === SPACE || + code === LINE_FEED || + code === TAB || + code === CARRIAGE_RETURN + ) +} diff --git a/services/web/shared/bibtex/parser/bibtex-stream-parser.mts b/services/web/shared/bibtex/parser/bibtex-stream-parser.mts new file mode 100644 index 0000000000..2347d5266a --- /dev/null +++ b/services/web/shared/bibtex/parser/bibtex-stream-parser.mts @@ -0,0 +1,792 @@ +import OError from '@overleaf/o-error' +import { BibtexEntry } from '../bibtex-entry.mts' +import { + BibtexFieldValue, + BibtexFieldValueBuilder, +} from '../bibtex-field-value.mts' +import { + AT, + CARRIAGE_RETURN, + CLOSE_BRACE, + CLOSE_PAREN, + COMMA, + EQUALS, + HASH, + LINE_FEED, + OPEN_BRACE, + OPEN_PAREN, + PERCENT, + QUOTE, + isBibtexIdentifierCharCode, + isBibtexIdentifierStartCharCode, + isDigitCode, + isWhitespaceCode, +} from './bibtex-char-codes.mts' + +export type BibtexEntryItem = { + kind: 'entry' + entry: BibtexEntry +} + +/** A `@string` definition, which the parser records rather than expands. */ +export type BibtexNamedStringItem = { + kind: 'namedString' + /** + * Kept as written, as are the references to it. Names are matched + * case-insensitively, so a consumer resolving references has to fold case + * when it builds its lookup table — but lowercasing here would rewrite the + * author's own spelling of the name whenever an entry is read and written + * back, which entry types and field names do not have to worry about. + * + * A definition also takes effect only from where it appears, replacing any + * earlier one from that point on, so resolution has to follow item order. + */ + name: string + value: BibtexFieldValue +} + +/** A `%` comment or a `@comment` body, without its delimiters. */ +export type BibtexCommentItem = { + kind: 'comment' + text: string +} + +/** Free-form text between entries, which BibTeX ignores. */ +export type BibtexJunkItem = { + kind: 'junk' + text: string +} + +export type BibtexItem = + | BibtexEntryItem + | BibtexNamedStringItem + | BibtexCommentItem + | BibtexJunkItem + +/* + * Parser states. Those up to and including AFTER_VALUE sit between entry + * tokens, where whitespace and %-comments are skipped; the main loop tests for + * them with a single comparison, so their order matters. The IN_* states are + * mid-token, and are the only ones a chunk boundary can fall inside. + */ +const TOP = 0 +const EXPECT_TYPE = 1 +const EXPECT_BODY_OPEN = 2 +const EXPECT_KEY = 3 +const AFTER_KEY = 4 +const EXPECT_FIELD_NAME = 5 +const EXPECT_EQUALS = 6 +const EXPECT_VALUE = 7 +const AFTER_VALUE = 8 +const LAST_INTER_TOKEN_STATE = AFTER_VALUE +const IN_TYPE = 9 +const IN_KEY = 10 +const IN_FIELD_NAME = 11 +const IN_NUMBER = 12 +const IN_NAME = 13 +const IN_LITERAL = 14 +const IN_COMMENT_BODY = 15 +const IN_JUNK_WORD = 16 +const IN_JUNK_SPACE = 17 +const IN_LINE_COMMENT = 18 + +/** What an entry's body holds, selected by its type. */ +const REGULAR = 0 // fields, introduced by a citation key +const NAMED_STRING = 1 // @string: named string definitions +const PREAMBLE = 2 // @preamble: a single value +const COMMENT = 3 // @comment: a literal + +/** Mirrors specializeEntryType in the editor's bibtex grammar. */ +function bodyKindOf(lowercasedType: string): number { + switch (lowercasedType) { + case 'string': + return NAMED_STRING + case 'preamble': + return PREAMBLE + case 'comment': + return COMMENT + default: + return REGULAR + } +} + +/** + * Cap on the input one item may span. Streaming only keeps memory down while + * items keep completing: an unterminated literal, a single enormous word of + * junk, or an entry with endless fields would otherwise buffer the whole rest of + * the input. The limit is far above any real entry, comment or run of junk, so + * reaching it means the input is not a .bib file. + */ +const MAX_ITEM_LENGTH = 1 << 20 + +/** + * Thrown when one item spans more than MAX_ITEM_LENGTH characters of input. It + * leaves the parser part-way through that item, so the parser has to be + * discarded rather than written to again. + */ +export class BibtexItemTooLargeError extends OError {} + +/** + * Chunk-oblivious BibTeX parser, built to index .bib files too large to hold a + * syntax tree for. It follows the same grammar as the editor's lezer parser + * (frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar) and reads + * every entry the same way, so a reader sees the same references the editor + * shows them. The differential test in the library module pins that. + * + * Items come out in the order they occur, with one exception: a comment written + * inside an entry is reported before the entry that contains it, because the + * entry is only complete at its closing delimiter. + * + * Entries sharing a citation key are all reported, leaving it to the caller to + * decide which one wins. `@preamble` bodies are the only content dropped. + * + * An item spanning more than MAX_ITEM_LENGTH characters of input throws + * BibtexItemTooLargeError, so that no input can buffer without bound. + */ +export class BibtexStreamParser { + private state = TOP + private commentReturnState = TOP + + /** + * Token text so far, which a chunk boundary may have cut in half. Only one + * token is ever in flight: every state that can start a comment or a run of + * junk is reached with the buffer already emptied. + */ + private token = '' + + /** Whitespace inside a run of junk, held back until a word follows it. */ + private pendingSpace = '' + + /** + * Input consumed towards the item being read, against MAX_ITEM_LENGTH. + * Everything the parser holds on to between chunks — the token in flight, the + * entry's type, key and fields, the parts of the value being built — was read + * from this stretch of input, so bounding it bounds all of them. + */ + private consumed = 0 + + private bodyKind = REGULAR + + /** Closing delimiter the entry body's opener calls for. */ + private bodyTerminator = CLOSE_BRACE + + private entryType: string | null = null + private entryKey: string | null = null + private fields = new Map() + private fieldName: string | null = null + + /** + * The parts of the value being read. An empty builder separates "foo" and + * "foo =", which define nothing — biber rejects both as syntax errors — from + * "foo = {}", which defines the empty string and so adds a part. + */ + private fieldValue = new BibtexFieldValueBuilder() + + /** Closing delimiter of the literal being scanned, and its brace nesting. */ + private literalTerminator = CLOSE_BRACE + private literalDepth = 0 + + private items: BibtexItem[] = [] + + /** Parses a chunk, returning the items it completed. */ + write(chunk: string): BibtexItem[] { + let i = 0 + while (i < chunk.length) { + const state = this.state + + if (state <= LAST_INTER_TOKEN_STATE) { + const code = chunk.charCodeAt(i) + if (isWhitespaceCode(code)) { + // Whitespace between tokens is dropped, so it costs no memory and + // goes uncounted: a file can hold as much of it as it likes. + i += 1 + continue + } + if (code === PERCENT) { + this.commentReturnState = state + this.state = IN_LINE_COMMENT + i += 1 + continue + } + } + + const start = i + + switch (state) { + case TOP: + i = this.parseTop(chunk, i) + break + case EXPECT_TYPE: + i = this.parseExpectType(chunk, i) + break + case EXPECT_BODY_OPEN: + i = this.parseExpectBodyOpen(chunk, i) + break + case EXPECT_KEY: + i = this.parseExpectKey(chunk, i) + break + case AFTER_KEY: + i = this.parseAfterKey(chunk, i) + break + case EXPECT_FIELD_NAME: + i = this.parseExpectFieldName(chunk, i) + break + case EXPECT_EQUALS: + i = this.parseExpectEquals(chunk, i) + break + case EXPECT_VALUE: + i = this.parseExpectValue(chunk, i) + break + case AFTER_VALUE: + i = this.parseAfterValue(chunk, i) + break + case IN_TYPE: + i = this.scanType(chunk, i) + break + case IN_KEY: + i = this.scanKey(chunk, i) + break + case IN_FIELD_NAME: + i = this.scanFieldName(chunk, i) + break + case IN_NUMBER: + i = this.scanNumber(chunk, i) + break + case IN_NAME: + i = this.scanName(chunk, i) + break + case IN_LITERAL: + i = this.scanValueLiteral(chunk, i) + break + case IN_COMMENT_BODY: + i = this.scanCommentBody(chunk, i) + break + case IN_JUNK_WORD: + i = this.scanJunkWord(chunk, i) + break + case IN_JUNK_SPACE: + i = this.scanJunkSpace(chunk, i) + break + case IN_LINE_COMMENT: + i = this.scanLineComment(chunk, i) + break + } + + this.consume(i - start) + } + return this.takeItems() + } + + /** + * Signals the end of the input, returning any final items. An entry left + * unterminated still yields what was parsed of it, as it does in the editor. + */ + end(): BibtexItem[] { + switch (this.state) { + case IN_TYPE: + this.finishType() + break + case IN_KEY: + this.finishKey() + break + case IN_FIELD_NAME: + this.finishFieldName() + break + case IN_NUMBER: + this.finishNumber() + break + case IN_NAME: + this.finishName() + break + case IN_LITERAL: + this.finishLiteral() + break + case IN_COMMENT_BODY: + this.emitComment() + break + case IN_JUNK_WORD: + case IN_JUNK_SPACE: + this.emitJunk() + break + case IN_LINE_COMMENT: + this.emitComment() + break + } + this.endEntry() + this.state = TOP + this.consumed = 0 + return this.takeItems() + } + + private parseTop(chunk: string, i: number): number { + if (chunk.charCodeAt(i) === AT) { + this.beginEntry() + this.state = EXPECT_TYPE + return i + 1 + } + this.state = IN_JUNK_WORD + return i + } + + /** + * Scans a word of free-form comment text. Where the word ends decides how the + * next "%" reads: it only opens a comment at the start of a word, so + * "Junk% @article{a}" holds an entry while "Junk % @article{a}" has it + * commented out. + */ + private scanJunkWord(chunk: string, i: number): number { + const start = i + while (i < chunk.length) { + const code = chunk.charCodeAt(i) + if (code === AT) { + this.token += chunk.slice(start, i) + this.emitJunk() + this.state = TOP + return i + } + if (isWhitespaceCode(code)) { + this.token += chunk.slice(start, i) + this.state = IN_JUNK_SPACE + return i + } + i += 1 + } + this.token += chunk.slice(start, i) + return i + } + + private scanJunkSpace(chunk: string, i: number): number { + const start = i + while (i < chunk.length && isWhitespaceCode(chunk.charCodeAt(i))) { + i += 1 + } + this.pendingSpace += chunk.slice(start, i) + if (i === chunk.length) { + return i + } + + const code = chunk.charCodeAt(i) + if (code === AT || code === PERCENT) { + this.emitJunk() + this.state = TOP + return i + } + this.token += this.pendingSpace + this.pendingSpace = '' + this.state = IN_JUNK_WORD + return i + } + + private scanLineComment(chunk: string, i: number): number { + const start = i + while (i < chunk.length) { + const code = chunk.charCodeAt(i) + if (code === LINE_FEED || code === CARRIAGE_RETURN) { + break + } + i += 1 + } + this.token += chunk.slice(start, i) + if (i < chunk.length) { + this.emitComment() + this.state = this.commentReturnState + } + return i + } + + private parseExpectType(chunk: string, i: number): number { + if (isBibtexIdentifierStartCharCode(chunk.charCodeAt(i))) { + this.state = IN_TYPE + return i + } + return this.recover(i) + } + + private scanType(chunk: string, i: number): number { + i = this.scanIdentifier(chunk, i) + if (i < chunk.length) { + this.finishType() + } + return i + } + + private finishType() { + const type = this.takeToken().toLowerCase() + this.entryType = type + this.bodyKind = bodyKindOf(type) + this.state = EXPECT_BODY_OPEN + } + + private parseExpectBodyOpen(chunk: string, i: number): number { + const code = chunk.charCodeAt(i) + if (code !== OPEN_BRACE && code !== OPEN_PAREN) { + return this.recover(i) + } + this.bodyTerminator = code === OPEN_BRACE ? CLOSE_BRACE : CLOSE_PAREN + switch (this.bodyKind) { + case COMMENT: + // A @comment body is a literal rather than a list of values, so it + // swallows any entry written inside it. + this.beginLiteral(this.bodyTerminator) + this.state = IN_COMMENT_BODY + break + case PREAMBLE: + this.state = EXPECT_VALUE + break + case NAMED_STRING: + this.state = EXPECT_FIELD_NAME + break + default: + this.state = EXPECT_KEY + } + return i + 1 + } + + private parseExpectKey(chunk: string, i: number): number { + if (isBibtexIdentifierCharCode(chunk.charCodeAt(i))) { + this.state = IN_KEY + return i + } + return this.recover(i) + } + + private scanKey(chunk: string, i: number): number { + i = this.scanIdentifier(chunk, i) + if (i < chunk.length) { + this.finishKey() + } + return i + } + + private finishKey() { + // Citation keys keep their case, unlike types and field names. + this.entryKey = this.takeToken() + this.state = AFTER_KEY + } + + private parseAfterKey(chunk: string, i: number): number { + const code = chunk.charCodeAt(i) + if (code === COMMA) { + this.state = EXPECT_FIELD_NAME + return i + 1 + } + if (code === this.bodyTerminator) { + return this.closeBody(i) + } + return this.recover(i) + } + + private parseExpectFieldName(chunk: string, i: number): number { + const code = chunk.charCodeAt(i) + if (code === this.bodyTerminator) { + return this.closeBody(i) + } + if (isBibtexIdentifierStartCharCode(code)) { + this.state = IN_FIELD_NAME + return i + } + return this.recover(i) + } + + private scanFieldName(chunk: string, i: number): number { + i = this.scanIdentifier(chunk, i) + if (i < chunk.length) { + this.finishFieldName() + } + return i + } + + private finishFieldName() { + this.fieldName = this.takeToken() + this.state = EXPECT_EQUALS + } + + private parseExpectEquals(chunk: string, i: number): number { + const code = chunk.charCodeAt(i) + if (code === EQUALS) { + this.state = EXPECT_VALUE + return i + 1 + } + if (code === this.bodyTerminator) { + return this.closeBody(i) + } + return this.recover(i) + } + + private parseExpectValue(chunk: string, i: number): number { + const code = chunk.charCodeAt(i) + if (code === OPEN_BRACE || code === QUOTE) { + this.beginLiteral(code === OPEN_BRACE ? CLOSE_BRACE : QUOTE) + this.state = IN_LITERAL + return i + 1 + } + if (isDigitCode(code)) { + this.state = IN_NUMBER + return i + } + if (isBibtexIdentifierStartCharCode(code)) { + this.state = IN_NAME + return i + } + // A missing value contributes no part, so "title=," and "title={x} #}" + // behave as if the value had already ended here. + this.state = AFTER_VALUE + return i + } + + private scanNumber(chunk: string, i: number): number { + const start = i + while (i < chunk.length && isDigitCode(chunk.charCodeAt(i))) { + i += 1 + } + this.token += chunk.slice(start, i) + if (i < chunk.length) { + this.finishNumber() + } + return i + } + + private finishNumber() { + this.fieldValue.addNumber(this.takeToken()) + this.state = AFTER_VALUE + } + + private scanName(chunk: string, i: number): number { + i = this.scanIdentifier(chunk, i) + if (i < chunk.length) { + this.finishName() + } + return i + } + + private finishName() { + // A reference to a named string is recorded, not expanded. + this.fieldValue.addNamedString(this.takeToken()) + this.state = AFTER_VALUE + } + + private scanValueLiteral(chunk: string, i: number): number { + i = this.scanLiteral(chunk, i) + if (i < chunk.length) { + this.finishLiteral() + return i + 1 + } + return i + } + + private finishLiteral() { + // A line break and the indentation following it collapse to one space. + this.fieldValue.addString(this.takeToken().replaceAll(/[\n\r]\s*/g, ' ')) + this.state = AFTER_VALUE + } + + private scanCommentBody(chunk: string, i: number): number { + i = this.scanLiteral(chunk, i) + if (i < chunk.length) { + this.emitComment() + this.endEntry() + this.state = TOP + return i + 1 + } + return i + } + + private parseAfterValue(chunk: string, i: number): number { + const code = chunk.charCodeAt(i) + if (code === HASH) { + this.state = EXPECT_VALUE + return i + 1 + } + if (code === COMMA) { + this.commitField() + this.state = EXPECT_FIELD_NAME + return i + 1 + } + if (code === this.bodyTerminator) { + return this.closeBody(i) + } + return this.recover(i) + } + + /** Appends identifier characters to the token buffer. */ + private scanIdentifier(chunk: string, i: number): number { + const start = i + while ( + i < chunk.length && + isBibtexIdentifierCharCode(chunk.charCodeAt(i)) + ) { + i += 1 + } + this.token += chunk.slice(start, i) + return i + } + + private beginLiteral(terminator: number) { + this.literalTerminator = terminator + this.literalDepth = 0 + } + + /** + * Appends literal text to the token buffer, stopping at its closing + * delimiter. Only braces nest, and there are no backslash escapes: every + * other delimiter is content until the nesting is back to zero. + */ + private scanLiteral(chunk: string, i: number): number { + const start = i + while (i < chunk.length) { + const code = chunk.charCodeAt(i) + if (code === OPEN_BRACE) { + this.literalDepth += 1 + } else if (code === CLOSE_BRACE && this.literalDepth > 0) { + this.literalDepth -= 1 + } else if (this.literalDepth === 0 && code === this.literalTerminator) { + break + } + i += 1 + } + this.token += chunk.slice(start, i) + return i + } + + /** + * Counts the input a step of the main loop read towards the item in progress. + * Landing back at TOP means the item is complete and the parser is holding + * nothing, so the count starts over from there. + */ + private consume(length: number) { + if (this.state === TOP) { + this.consumed = 0 + return + } + this.consumed += length + if (this.consumed > MAX_ITEM_LENGTH) { + throw new BibtexItemTooLargeError('bibtex item is too large', { + limit: MAX_ITEM_LENGTH, + }) + } + } + + private takeToken(): string { + const token = this.token + this.token = '' + return token + } + + private emitJunk() { + const text = this.takeToken() + this.pendingSpace = '' + if (text !== '') { + this.items.push({ kind: 'junk', text }) + } + } + + private emitComment() { + const text = this.takeToken() + // A comment can sit inside an entry, whose fields are still held, so only + // the comment's own share of the input is given back. + this.consumed -= text.length + this.items.push({ kind: 'comment', text }) + } + + private beginEntry() { + this.bodyKind = REGULAR + this.entryType = null + this.entryKey = null + this.fields = new Map() + this.fieldName = null + this.fieldValue = new BibtexFieldValueBuilder() + this.token = '' + } + + private commitField() { + if (this.fieldName != null) { + if (this.bodyKind === NAMED_STRING) { + // "@string{foo}" and "@string{foo =}" define nothing, so reporting them + // would let either shadow an earlier valid definition of foo. + if (!this.fieldValue.isEmpty()) { + this.items.push({ + kind: 'namedString', + name: this.fieldName, + value: this.fieldValue.build(), + }) + } + } else { + // A field with no value is still committed, to match the editor. + this.fields.set(this.fieldName.toLowerCase(), this.fieldValue.build()) + } + } + this.fieldName = null + this.fieldValue = new BibtexFieldValueBuilder() + } + + private closeBody(i: number): number { + this.endEntry() + this.state = TOP + return i + 1 + } + + /** + * Ends the entry where it stands after an unexpected character, keeping the + * fields parsed so far. Leaving the character for the top level to look at + * again is what lets an unclosed entry hand the next "@" to a fresh one. + */ + private recover(i: number): number { + this.endEntry() + this.state = TOP + return i + } + + /** + * Completes the entry in progress, reporting it if it has both a type and a + * citation key. + */ + private endEntry() { + this.commitField() + if ( + this.bodyKind === REGULAR && + this.entryType != null && + this.entryKey != null + ) { + this.items.push({ + kind: 'entry', + entry: new BibtexEntry({ + type: this.entryType, + key: this.entryKey, + fields: this.fields, + }), + }) + } + this.entryType = null + this.entryKey = null + this.fields = new Map() + this.token = '' + } + + private takeItems(): BibtexItem[] { + const items = this.items + this.items = [] + return items + } +} + +/** Parses a whole .bib file held in memory. */ +export function parseBibtexItems(text: string): BibtexItem[] { + const parser = new BibtexStreamParser() + const items = parser.write(text) + return items.concat(parser.end()) +} + +/** + * Parses a .bib file arriving in chunks. The caller is responsible for decoding + * bytes with a stateful decoder, so a multi-byte character split across chunks + * is not corrupted. + */ +export async function* streamBibtexItems( + chunks: AsyncIterable +): AsyncGenerator { + const parser = new BibtexStreamParser() + for await (const chunk of chunks) { + yield* parser.write(chunk) + } + yield* parser.end() +}