Merge pull request #33162 from overleaf/cd-upload-generated-files-to-filetree

Cd upload generated files to filetree

GitOrigin-RevId: 2d2774e57c42452fba3a2582fde7153ffcde59bf

services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts

@@ -1,10 +1,12 @@
+import path from 'path-browserify'
 import type {
   OutputStream,
-  OutputFileData,
   ProjectFileData,
   PyodideWorkerRequest,
   PyodideWorkerResponse,
 } from './pyodide-worker-messages'
+import type { BatchUploadItem } from '@/infrastructure/batch-file-uploader'
+import type { FileUploader } from './python-runner'
 
 export type OutputCallback = (
   stream: OutputStream,
@@ -23,7 +25,7 @@ export type LifecycleCallback = (
         executionId: string
         success: boolean
         outputs: string[]
-        outputFiles: OutputFileData[]
+        failedUploads: string[]
       }
 ) => void
 
@@ -37,17 +39,20 @@ export class PyodideWorkerClient {
   private pendingMessages: PyodideWorkerRequest[] = []
   private outputCallback: OutputCallback | null
   private lifecycleCallback: LifecycleCallback | null
+  private fileUploader: FileUploader
 
   constructor(options: {
     baseAssetPath: string
     createWorker: () => Worker
     onOutput?: OutputCallback
     onLifecycle?: LifecycleCallback
+    fileUploader: FileUploader
   }) {
     this.baseAssetPath = options.baseAssetPath
     this.createWorker = options.createWorker
     this.outputCallback = options.onOutput ?? null
     this.lifecycleCallback = options.onLifecycle ?? null
+    this.fileUploader = options.fileUploader
     this.worker = this.createWorker()
     this.worker.addEventListener('message', this.receive)
 
@@ -119,7 +124,7 @@ export class PyodideWorkerClient {
     }
   }
 
-  private receive = (event: MessageEvent<PyodideWorkerResponse>) => {
+  private receive = async (event: MessageEvent<PyodideWorkerResponse>) => {
     // Discard messages from a previously terminated worker
     if (event.target !== this.worker) {
       return
@@ -158,15 +163,55 @@ export class PyodideWorkerClient {
         )
         return
 
-      case 'run-code-result':
+      case 'run-code-result': {
+        let success = response.success
+        const failedUploads: string[] = []
+
+        if (success && response.outputFiles.length > 0) {
+          const items: BatchUploadItem[] = response.outputFiles.map(file => ({
+            file: new Blob([file.content as Uint8Array<ArrayBuffer>]),
+            name: path.basename(file.relativePath),
+            relativePath: file.relativePath,
+          }))
+
+          try {
+            const results = await this.fileUploader(items)
+            for (const result of results) {
+              if (result.status === 'error') {
+                failedUploads.push(result.relativePath!)
+                this.outputCallback?.(
+                  'stderr',
+                  `Failed to upload output file ${result.relativePath!}: ${result.error}`,
+                  response.fileId,
+                  response.executionId
+                )
+              }
+            }
+            if (failedUploads.length > 0) {
+              success = false
+            }
+          } catch (err) {
+            const message = err instanceof Error ? err.message : String(err)
+            this.outputCallback?.(
+              'stderr',
+              `Failed to upload output files: ${message}`,
+              response.fileId,
+              response.executionId
+            )
+            failedUploads.push(...items.map(item => item.relativePath!))
+            success = false
+          }
+        }
+
         this.lifecycleCallback?.({
           type: 'run-finished',
           fileId: response.fileId,
           executionId: response.executionId,
-          success: response.success,
+          success,
           outputs: response.outputs,
-          outputFiles: response.outputFiles,
+          failedUploads,
         })
+      }
     }
   }
 }

services/web/frontend/js/features/ide-react/components/editor/python/python-runner.ts

@@ -5,6 +5,12 @@ import { v4 as uuid } from 'uuid'
 import { debugConsole } from '@/utils/debugging'
 import { PyodideWorkerClient } from './pyodide-worker-client'
 import type { OutputStream } from './pyodide-worker-messages'
+import type {
+  BatchUploadItem,
+  UploadResult,
+} from '@/infrastructure/batch-file-uploader'
+
+export type FileUploader = (items: BatchUploadItem[]) => Promise<UploadResult[]>
 
 const MAX_OUTPUT_LINES = 100
 
@@ -45,6 +51,8 @@ export class PythonRunner {
   private readonly baseAssetPath: string
   private readonly createWorker: () => Worker
   private readonly getExecutionContext: () => Promise<ExecutionContext | null>
+  private readonly fileUploader: FileUploader
+
   private listeners = new Set<Listener>()
 
   private activeExecutionId: string | null = null
@@ -54,12 +62,14 @@ export class PythonRunner {
     fileId: string,
     baseAssetPath: string,
     getExecutionContext: () => Promise<ExecutionContext | null>,
-    createWorker: () => Worker
+    createWorker: () => Worker,
+    fileUploader: FileUploader
   ) {
     this.fileId = fileId
     this.baseAssetPath = baseAssetPath
     this.createWorker = createWorker
     this.getExecutionContext = getExecutionContext
+    this.fileUploader = fileUploader
   }
 
   subscribe = (listener: Listener): (() => void) => {
@@ -100,6 +110,7 @@ export class PythonRunner {
     this.client = new PyodideWorkerClient({
       baseAssetPath: this.baseAssetPath,
       createWorker: this.createWorker,
+      fileUploader: this.fileUploader,
       onLifecycle: event => {
         switch (event.type) {
           case 'loaded':
@@ -111,15 +122,17 @@ export class PythonRunner {
             this.updateState({ status: 'errored', error: event.error })
             return
 
-          case 'run-finished':
+          case 'run-finished': {
             if (
               event.fileId !== this.fileId ||
               this.activeExecutionId !== event.executionId
             ) {
               return
             }
+
             this.activeExecutionId = null
             this.updateState({ status: 'finished' })
+          }
         }
       },
       onOutput: (stream, line, fileId, executionId) => {

services/web/frontend/js/features/ide-react/context/python-execution-context.tsx

@@ -12,6 +12,11 @@ import getMeta from '@/utils/meta'
 import { useFileTreePathContext } from '@/features/file-tree/contexts/file-tree-path'
 import { useEditorManagerContext } from '@/features/ide-react/context/editor-manager-context'
 import { useProjectContext } from '@/shared/context/project-context'
+import { useFileTreeData } from '@/shared/context/file-tree-data-context'
+import {
+  uploadBatch,
+  BatchUploadItem,
+} from '@/infrastructure/batch-file-uploader'
 import {
   PythonRunner,
   ExecutionContext,
@@ -41,14 +46,22 @@ export const PythonExecutionProvider: FC<PropsWithChildren> = ({
   children,
 }) => {
   const { openDocs } = useEditorManagerContext()
-  const { projectSnapshot } = useProjectContext()
+  const { projectId, projectSnapshot } = useProjectContext()
   const { pathInFolder } = useFileTreePathContext()
+  const { fileTreeData } = useFileTreeData()
   const runnersRef = useRef(new Map<string, PythonRunner>())
   const baseAssetPathRef = useRef<string | null>(null)
 
   const pathInFolderRef = useRef(pathInFolder)
   pathInFolderRef.current = pathInFolder
 
+  // Ref so the upload closure built into each PythonRunner reads the
+  // current value at call time rather than capturing a potentially-stale
+  // value from when the runner was constructed (fileTreeData may load
+  // after the runner is created).
+  const fileTreeDataRef = useRef(fileTreeData)
+  fileTreeDataRef.current = fileTreeData
+
   // Refreshes the project snapshot and resolves the source code and all project
   // files for the given fileId, to be passed to the executor for running.
   const getExecutionContext = useCallback(
@@ -95,17 +108,31 @@ export const PythonExecutionProvider: FC<PropsWithChildren> = ({
         ).toString()
       }
 
+      const uploadOutputFiles = (items: BatchUploadItem[]) => {
+        const folderId = fileTreeDataRef.current?._id
+        if (!folderId) {
+          return Promise.reject(
+            new Error('File tree not loaded; cannot upload output files')
+          )
+        }
+        return uploadBatch(items, {
+          projectId,
+          folderId,
+        })
+      }
+
       const runner = new PythonRunner(
         fileId,
         baseAssetPathRef.current,
         () => getExecutionContext(fileId),
-        createPyodideWorker
+        createPyodideWorker,
+        uploadOutputFiles
       )
       runner.init()
       runnersRef.current.set(fileId, runner)
       return runner
     },
-    [getExecutionContext]
+    [getExecutionContext, projectId]
   )
 
   useEffect(() => {