Merge pull request #24468 from overleaf/mj-client-side-references

[web] Perform ARS on client-side

GitOrigin-RevId: 19703c82758cae450fe52463ad9612d3a2383ba0

services/web/frontend/js/features/ide-react/references/basic-reference-index.ts

@@ -0,0 +1,24 @@
+import { ReferenceIndex } from './reference-index'
+import { Changes } from './types'
+
+export default class BasicReferenceIndex extends ReferenceIndex {
+  fileIndex: Map<string, Set<string>> = new Map()
+
+  updateIndex({ updates, deletes }: Changes): Set<string> {
+    for (const path of deletes) {
+      this.fileIndex.delete(path)
+    }
+    for (const { path, content } of updates) {
+      const fileReferences: Set<string> = new Set()
+      const entries = this.parseEntries(content)
+      for (const entry of entries) {
+        fileReferences.add(entry.EntryKey)
+      }
+      this.fileIndex.set(path, fileReferences)
+    }
+    this.keys = new Set(
+      this.fileIndex.values().flatMap(entry => Array.from(entry))
+    )
+    return this.keys
+  }
+}

services/web/frontend/js/features/ide-react/references/reference-index.ts

@@ -0,0 +1,43 @@
+import Bib2Json from './bib2json'
+import { AdvancedReferenceSearchResult, Bib2JsonEntry, Changes } from './types'
+
+export abstract class ReferenceIndex {
+  keys: Set<string> = new Set()
+
+  abstract updateIndex({ updates, deletes }: Changes): void
+  async search(_query: string): Promise<AdvancedReferenceSearchResult> {
+    return { hits: [] }
+  }
+
+  getKeys(): Set<string> {
+    return this.keys
+  }
+
+  parseEntries(content: string): Bib2JsonEntry[] {
+    const allowedFields = ['author', 'journal', 'title', 'year', 'date']
+    // @ts-expect-error Bib2Json works as both a constructor and a function
+    const { entries } = Bib2Json(content, allowedFields)
+    for (const entry of entries) {
+      if (entry.Fields?.year) {
+        entry.Fields.year = parseInt(entry.Fields.year).toString()
+        if (entry.Fields.year === 'NaN') {
+          delete entry.Fields.year
+        }
+      }
+      setDefaultFields(entry.Fields)
+    }
+    return entries
+  }
+}
+
+function setDefaultFields(
+  fields: Partial<Bib2JsonEntry['Fields']>
+): Bib2JsonEntry['Fields'] {
+  const requiredFields = ['author', 'journal', 'title', 'date', 'year'] as const
+  for (const field of requiredFields) {
+    if (!fields[field]) {
+      fields[field] = ''
+    }
+  }
+  return fields as Bib2JsonEntry['Fields']
+}

services/web/frontend/js/features/ide-react/references/reference-indexer.ts

@@ -0,0 +1,137 @@
+import { ProjectSnapshot } from '@/infrastructure/project-snapshot'
+import { generateSHA1Hash } from '@/shared/utils/sha1'
+import { AdvancedReferenceSearchResult, Changes } from './types'
+import { debugConsole } from '@/utils/debugging'
+import type { ReferenceWorkerResponse } from './references.worker'
+
+const ONE_MB = 1024 * 1024
+const MAX_BIB_DATA_SIZE = 6 * ONE_MB
+
+export class ReferenceIndexer {
+  private fileIndexHash: Map<string, string> = new Map()
+  private worker: Worker
+  private updateResolve: ((result: Set<string>) => void) | null = null
+  private searchResolve:
+    | ((result: AdvancedReferenceSearchResult) => void)
+    | null = null
+
+  constructor() {
+    this.worker = new Worker(
+      /* webpackChunkName: "references-worker" */
+      new URL('./references.worker.ts', import.meta.url),
+      { type: 'module' }
+    )
+    this.worker.addEventListener('message', evt => this.handleMessage(evt))
+  }
+
+  private handleMessage(event: MessageEvent) {
+    const data = event.data as ReferenceWorkerResponse
+    if (data.type === 'searchResult' && this.searchResolve) {
+      this.searchResolve(data.result)
+      this.searchResolve = null
+    } else if (data.type === 'updateKeys' && this.updateResolve) {
+      this.updateResolve(data.keys)
+      this.updateResolve = null
+    } else {
+      debugConsole.warn('Received unknown message from worker:', data.type)
+    }
+  }
+
+  async updateFromSnapshot(
+    snapshot: Pick<
+      ProjectSnapshot,
+      | 'getDocPaths'
+      | 'getDocContents'
+      | 'getBinaryFilePathsWithHash'
+      | 'getBinaryFileContents'
+    >,
+    {
+      dataLimit = MAX_BIB_DATA_SIZE,
+      signal,
+    }: { dataLimit?: number; signal: AbortSignal }
+  ): Promise<Set<string>> {
+    const nextFileHashIndex = new Map(this.fileIndexHash)
+    const previousPaths = new Set(this.fileIndexHash.keys())
+    let dataBudget = dataLimit
+    const docs = snapshot
+      .getDocPaths()
+      .filter(path => path.toLowerCase().endsWith('.bib'))
+
+    const changes: Changes = { updates: [], deletes: [] }
+    for (const path of docs) {
+      previousPaths.delete(path)
+      if (dataBudget <= 0) {
+        continue
+      }
+      const content = snapshot.getDocContents(path)?.slice(0, dataBudget)
+      if (content == null) {
+        continue
+      }
+      dataBudget -= content.length
+      const hash = generateSHA1Hash(content)
+      const possibleMatch = nextFileHashIndex.get(path)
+      if (possibleMatch === undefined || possibleMatch !== hash) {
+        // New or changed file
+        nextFileHashIndex.set(path, hash)
+        changes.updates.push({ path, content })
+      }
+    }
+
+    const files = snapshot
+      .getBinaryFilePathsWithHash()
+      .filter(({ path }) => path.toLowerCase().endsWith('.bib'))
+      .sort((a, b) => a.size - b.size)
+
+    for (const { path, hash, size } of files) {
+      if (signal.aborted) {
+        debugConsole.warn('Aborted indexing references due to signal')
+        return new Set()
+      }
+
+      previousPaths.delete(path)
+      if (nextFileHashIndex.get(path) === hash) {
+        dataBudget -= size
+        // Already indexed
+        continue
+      }
+      if (dataBudget <= 0) {
+        continue
+      }
+      const content = await snapshot.getBinaryFileContents(path, {
+        maxSize: dataBudget,
+      })
+      dataBudget -= content.length
+      nextFileHashIndex.set(path, hash)
+      changes.updates.push({ path, content })
+    }
+
+    previousPaths.forEach(path => {
+      // Deleted file
+      changes.deletes.push(path)
+      nextFileHashIndex.delete(path)
+    })
+
+    if (dataBudget <= 0) {
+      debugConsole.warn('Data budget exceeded while updating references index')
+    }
+
+    this.fileIndexHash = nextFileHashIndex
+
+    this.worker.postMessage({
+      type: 'update',
+      changes,
+    })
+
+    return new Promise(resolve => {
+      this.updateResolve = resolve
+    })
+  }
+
+  async search(query: string): Promise<AdvancedReferenceSearchResult> {
+    this.worker.postMessage({ type: 'search', query })
+    const { promise, resolve } =
+      Promise.withResolvers<AdvancedReferenceSearchResult>()
+    this.searchResolve = resolve
+    return promise
+  }
+}

services/web/frontend/js/features/ide-react/references/references.worker.ts

@@ -0,0 +1,47 @@
+import BasicReferenceIndex from './basic-reference-index'
+import { ReferenceIndex } from './reference-index'
+import { AdvancedReferenceSearchResult, Changes } from './types'
+import importOverleafModules from '../../../../macros/import-overleaf-module.macro'
+
+interface IndexConstructor {
+  new (): ReferenceIndex
+}
+
+const indices = importOverleafModules('referenceIndices') as {
+  import: { default: IndexConstructor }
+  path: string
+}[]
+
+export type ReferenceWorkerRequest =
+  | { type: 'update'; changes: Changes }
+  | { type: 'search'; query: string }
+
+export type ReferenceWorkerResponse =
+  | { type: 'updateKeys'; keys: Set<string> }
+  | { type: 'searchResult'; result: AdvancedReferenceSearchResult }
+
+function createIndex(): ReferenceIndex {
+  const Klass = indices[0]?.import.default ?? BasicReferenceIndex
+  return new Klass()
+}
+
+const indexer: ReferenceIndex = createIndex()
+
+self.addEventListener('message', async (event: MessageEvent) => {
+  const message = event.data as ReferenceWorkerRequest
+  switch (message.type) {
+    case 'update':
+      indexer.updateIndex(message.changes)
+      self.postMessage({ type: 'updateKeys', keys: indexer.getKeys() })
+      break
+
+    case 'search': {
+      const result = await indexer.search(message.query)
+      self.postMessage({ type: 'searchResult', result })
+      break
+    }
+
+    default:
+      console.error('Unknown message type:', message)
+  }
+})

services/web/frontend/js/features/ide-react/references/types.ts

@@ -0,0 +1,23 @@
+export type Bib2JsonEntry = {
+  EntryKey: string
+  Fields: {
+    author: string
+    date: string
+    journal: string
+    title: string
+    year: string
+  }
+}
+
+export type AdvancedReferenceSearchResult = {
+  hits: {
+    _source: Bib2JsonEntry
+  }[]
+}
+
+export type ReferenceEntry = Map<string, Bib2JsonEntry>
+
+export type Changes = {
+  updates: { path: string; content: string }[]
+  deletes: string[]
+}