Add read_project_outline tool to the AI assistant (#31875)

GitOrigin-RevId: 37e7d9bebdc6291a0ea7d89b73494d24f5cd5908

services/web/frontend/js/features/source-editor/utils/tree-operations/outline.ts

@@ -108,6 +108,50 @@ const getEntryText = (state: EditorState, node: SyntaxNodeRef): string => {
   return titleParts.join('')
 }
 
+export const readSectioningCommandOutlineItem = (
+  state: EditorState,
+  command: SyntaxNode
+): FlatOutlineItem | false | undefined => {
+  const name = command.getChild('SectioningArgument')?.getChild('LongArg')
+
+  if (!name) {
+    return
+  }
+
+  const parent = command.parent
+
+  // Filter out descendants of newcommand/renewcommand
+  for (
+    let ancestor: SyntaxNode | null = parent;
+    ancestor;
+    ancestor = ancestor.parent
+  ) {
+    if (ancestor.type.is('NewCommand') || ancestor.type.is('RenewCommand')) {
+      return false
+    }
+  }
+
+  const getCommandName = () => {
+    const ctrlSeq = command.firstChild
+    if (!ctrlSeq) return ''
+    // Ignore the \
+    return state.doc.sliceString(ctrlSeq.from + 1, ctrlSeq.to)
+  }
+
+  const nestingLevel = parent?.type.is('$Section')
+    ? getNestingLevel(parent.type.id)
+    : getNestingLevel(getCommandName())
+
+  return {
+    line: state.doc.lineAt(command.from).number,
+    toLine: state.doc.lineAt(command.to).number,
+    title: getEntryText(state, name),
+    from: command.from,
+    to: command.to,
+    level: nestingLevel,
+  }
+}
+
 /**
  * Extracts FlatOutlineItem instances from the syntax tree
  */
@@ -118,51 +162,19 @@ export const enterNode = (
   nodeIntersectsChange: NodeIntersectsChangeFn
 ): any => {
   if (node.type.is('SectioningCommand')) {
-    const command = node.node
-    const parent = command.parent
-
-    if (!nodeIntersectsChange(command)) {
+    if (!nodeIntersectsChange(node)) {
       // This should already be in `items`
       return
     }
-    const name = command.getChild('SectioningArgument')?.getChild('LongArg')
 
-    if (!name) {
+    const titleNode = readSectioningCommandOutlineItem(state, node.node)
+
+    if (typeof titleNode === 'object') {
+      items.push(titleNode)
       return
     }
 
-    // Filter out descendants of newcommand/renewcommand
-    for (
-      let ancestor: SyntaxNode | null = parent;
-      ancestor;
-      ancestor = ancestor.parent
-    ) {
-      if (ancestor.type.is('NewCommand') || ancestor.type.is('RenewCommand')) {
-        return false
-      }
-    }
-
-    const getCommandName = () => {
-      const ctrlSeq = command.firstChild
-      if (!ctrlSeq) return ''
-      // Ignore the \
-      return state.doc.sliceString(ctrlSeq.from + 1, ctrlSeq.to)
-    }
-
-    const nestingLevel = parent?.type.is('$Section')
-      ? getNestingLevel(parent.type.id)
-      : getNestingLevel(getCommandName())
-
-    const thisNode = {
-      line: state.doc.lineAt(command.from).number,
-      toLine: state.doc.lineAt(command.to).number,
-      title: getEntryText(state, name),
-      from: command.from,
-      to: command.to,
-      level: nestingLevel,
-    }
-
-    items.push(thisNode)
+    return titleNode // false to skip iterating over this node's children, or undefined
   }
   if (node.type.is('$Environment')) {
     const environmentNode = node.node

services/web/frontend/js/features/source-editor/utils/tree-operations/projection.ts

@@ -77,7 +77,7 @@ export type EnterNodeFn<T> = (
   node: SyntaxNodeRef,
   items: T[],
   nodeIntersectsChange: NodeIntersectsChangeFn
-) => any
+) => boolean | void
 
 /**
  * Calculates an updated projection of an editor state. Passing a previous ProjectionResult

services/web/frontend/js/features/source-editor/utils/walk-included-files.ts

@@ -0,0 +1,87 @@
+import path from 'path-browserify'
+import { SyntaxNodeRef, Tree } from '@lezer/common'
+import { LaTeXLanguage } from '@/features/source-editor/languages/latex/latex-language'
+import { ProjectSnapshot } from '@/infrastructure/project-snapshot'
+import { debugConsole } from '@/utils/debugging'
+
+export type IncludedFile = {
+  docPath: string
+  content: string
+  tree: Tree
+}
+
+export type IncludedFileSnapshot = Pick<
+  ProjectSnapshot,
+  'locateFile' | 'getDocContents'
+>
+
+/**
+ * Read the file path from an IncludeArgument/InputArgument/SubfileArgument
+ * node, handling both the braced form (`\input{file}`) and the brace-less
+ * form (`\input file`).
+ */
+export const readIncludedFilePath = (
+  content: string,
+  nodeRef: SyntaxNodeRef
+): string | undefined => {
+  const argument =
+    nodeRef.node.getChild('FilePathArgument')?.getChild('LiteralArgContent') ??
+    nodeRef.node
+      .getChild('BareFilePathArgument')
+      ?.getChild('SpaceDelimitedLiteralArgContent')
+
+  if (argument) {
+    return content.substring(argument.from, argument.to).trim() || undefined
+  }
+}
+
+/**
+ * Walk the tree of files reachable from relativePath via
+ * \include/\input/\subfile, parsing each file and calling `visit` in document
+ * order. The visitor receives a `recurse` callback to descend into an
+ * included file when it encounters one; included paths are resolved relative
+ * to the directory of the including file. Include cycles are ignored.
+ */
+export const walkIncludedFiles = (
+  projectSnapshot: IncludedFileSnapshot,
+  relativePath: string,
+  basePath: string,
+  visit: (file: IncludedFile, recurse: (includePath: string) => void) => void
+) => {
+  // the stack of files currently being visited, for detecting include cycles
+  const visiting = new Set<string>()
+
+  const walk = (filePath: string, currentBasePath: string) => {
+    const docPath = projectSnapshot.locateFile(filePath, currentBasePath)
+    if (!docPath) {
+      debugConsole.warn(`Couldn't find ${filePath} from ${currentBasePath}`)
+      return
+    }
+
+    if (visiting.has(docPath)) {
+      return
+    }
+
+    const content = projectSnapshot.getDocContents(docPath)
+    if (!content) {
+      debugConsole.warn(`No doc content in ${docPath}`)
+      return
+    }
+
+    const tree = LaTeXLanguage.parser.parse(content)
+
+    // locateFile expects an absolute directory as its base path
+    const docDir = path.dirname(`/${docPath}`)
+
+    visiting.add(docPath)
+    try {
+      visit({ docPath, content, tree }, includePath =>
+        walk(includePath, docDir)
+      )
+    } finally {
+      visiting.delete(docPath)
+    }
+  }
+
+  walk(relativePath, basePath)
+}

services/web/frontend/js/features/word-count-modal/utils/count-words-in-file.ts

@@ -1,10 +1,14 @@
-import { LaTeXLanguage } from '@/features/source-editor/languages/latex/latex-language'
 import { WordCountData } from '@/features/word-count-modal/components/word-count-data'
 import { NodeType, SyntaxNodeRef } from '@lezer/common'
 import { debugConsole } from '@/utils/debugging'
 import { findPreambleExtent } from '@/features/word-count-modal/utils/find-preamble-extent'
 import { Segmenters } from './segmenters'
-import { ProjectSnapshot } from '@/infrastructure/project-snapshot'
+import {
+  IncludedFile,
+  IncludedFileSnapshot,
+  readIncludedFilePath,
+  walkIncludedFiles,
+} from '@/features/source-editor/utils/walk-included-files'
 
 // const whiteSpaceRe = /^\s$/
 
@@ -109,28 +113,24 @@ type TextNode = {
 
 export const countWordsInFile = (
   data: WordCountData,
-  projectSnapshot: ProjectSnapshot,
+  projectSnapshot: IncludedFileSnapshot,
   relativePath: string,
   basePath: string,
   segmenters: Segmenters
 ) => {
-  const docPath = projectSnapshot.locateFile(relativePath, basePath)
-  if (!docPath) {
-    debugConsole.warn(`Couldn't find ${relativePath} from ${basePath}`)
-    return
-  }
-
-  const content = projectSnapshot.getDocContents(docPath)
-  if (!content) {
-    debugConsole.warn(`No doc content in ${docPath}`)
-    return
-  }
+  walkIncludedFiles(projectSnapshot, relativePath, basePath, (file, recurse) =>
+    countWordsInParsedFile(data, file, recurse, segmenters)
+  )
+}
 
+const countWordsInParsedFile = (
+  data: WordCountData,
+  { docPath, content, tree }: IncludedFile,
+  recurse: (includePath: string) => void,
+  segmenters: Segmenters
+) => {
   debugConsole.log(`Counting words in ${docPath}`)
 
-  // TODO: language from file extension
-  const tree = LaTeXLanguage.parser.parse(content)
-
   let currentContext: Context = 'text'
 
   const textNodes: TextNode[] = []
@@ -255,10 +255,10 @@ export const countWordsInFile = (
       return false
     },
     'IncludeArgument InputArgument SubfileArgument'(nodeRef) {
-      const path = content.substring(nodeRef.from + 1, nodeRef.to - 1)
-      debugConsole.log(path)
-      if (path) {
-        countWordsInFile(data, projectSnapshot, path, docPath, segmenters)
+      const includedPath = readIncludedFilePath(content, nodeRef)
+      debugConsole.log(includedPath)
+      if (includedPath) {
+        recurse(includedPath)
       }
     },
     'BlankLine LineBreak'(nodeRef) {