mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-15 06:33:55 +01:00
Merge pull request #34363 from overleaf/cd-migrate-python-runner-to-modules
Migrate the Python script runner into a closed python-runner module GitOrigin-RevId: ac22f45bcf744c693591076f0b41e8d52249dcbf
This commit is contained in:
Chris Dryden
-225
@@ -1,225 +0,0 @@
|
||||
import path from 'path-browserify'
|
||||
import type {
|
||||
ExecutionErrorType,
|
||||
OutputStream,
|
||||
ProjectFileData,
|
||||
PyodideWorkerRequest,
|
||||
PyodideWorkerResponse,
|
||||
} from './pyodide-worker-messages'
|
||||
import type { BatchUploadItem } from '@/infrastructure/batch-file-uploader'
|
||||
import type { FileUploader } from './python-runner'
|
||||
|
||||
export type OutputCallback = (
|
||||
stream: OutputStream,
|
||||
line: string,
|
||||
fileId: string,
|
||||
executionId: string
|
||||
) => void
|
||||
|
||||
export type LifecycleCallback = (
|
||||
event:
|
||||
| { type: 'loaded' }
|
||||
| { type: 'loading-failed'; error: string }
|
||||
| {
|
||||
type: 'run-finished'
|
||||
fileId: string
|
||||
executionId: string
|
||||
success: boolean
|
||||
outputs: string[]
|
||||
failedUploads: string[]
|
||||
imports: string[]
|
||||
errorType?: ExecutionErrorType
|
||||
}
|
||||
) => void
|
||||
|
||||
export class PyodideWorkerClient {
|
||||
private worker: Worker
|
||||
private baseAssetPath: string
|
||||
private createWorker: () => Worker
|
||||
private listening = false
|
||||
private destroyed = false
|
||||
private loadingError: string | null = null
|
||||
private pendingMessages: PyodideWorkerRequest[] = []
|
||||
private outputCallback: OutputCallback | null
|
||||
private lifecycleCallback: LifecycleCallback | null
|
||||
private fileUploader: FileUploader
|
||||
|
||||
constructor(options: {
|
||||
baseAssetPath: string
|
||||
createWorker: () => Worker
|
||||
onOutput?: OutputCallback
|
||||
onLifecycle?: LifecycleCallback
|
||||
fileUploader: FileUploader
|
||||
}) {
|
||||
this.baseAssetPath = options.baseAssetPath
|
||||
this.createWorker = options.createWorker
|
||||
this.outputCallback = options.onOutput ?? null
|
||||
this.lifecycleCallback = options.onLifecycle ?? null
|
||||
this.fileUploader = options.fileUploader
|
||||
this.worker = this.createWorker()
|
||||
this.worker.addEventListener('message', this.receive)
|
||||
|
||||
this.queueMessage({
|
||||
type: 'init',
|
||||
baseAssetPath: this.baseAssetPath,
|
||||
})
|
||||
}
|
||||
|
||||
runCode(
|
||||
code: string,
|
||||
options: { fileId: string; executionId: string; files: ProjectFileData[] }
|
||||
): void {
|
||||
if (this.destroyed) {
|
||||
throw new Error('Pyodide worker client has been destroyed')
|
||||
}
|
||||
|
||||
if (this.loadingError) {
|
||||
throw new Error(this.loadingError)
|
||||
}
|
||||
|
||||
this.queueMessage({
|
||||
type: 'run-code',
|
||||
code,
|
||||
fileId: options.fileId,
|
||||
executionId: options.executionId,
|
||||
files: options.files,
|
||||
})
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
if (this.destroyed) {
|
||||
return
|
||||
}
|
||||
|
||||
// Terminate the current worker immediately
|
||||
this.worker.terminate()
|
||||
this.pendingMessages.length = 0
|
||||
|
||||
// Reset state for the new worker
|
||||
this.listening = false
|
||||
this.loadingError = null
|
||||
|
||||
// Create a fresh worker and re-initialize Pyodide
|
||||
this.worker = this.createWorker()
|
||||
this.worker.addEventListener('message', this.receive)
|
||||
this.queueMessage({
|
||||
type: 'init',
|
||||
baseAssetPath: this.baseAssetPath,
|
||||
})
|
||||
}
|
||||
|
||||
destroy() {
|
||||
if (this.destroyed) {
|
||||
return
|
||||
}
|
||||
|
||||
this.destroyed = true
|
||||
this.pendingMessages.length = 0
|
||||
|
||||
this.worker.terminate()
|
||||
}
|
||||
|
||||
private queueMessage(message: PyodideWorkerRequest) {
|
||||
if (this.listening) {
|
||||
this.worker.postMessage(message)
|
||||
} else {
|
||||
this.pendingMessages.push(message)
|
||||
}
|
||||
}
|
||||
|
||||
private receive = async (event: MessageEvent<PyodideWorkerResponse>) => {
|
||||
// Discard messages from a previously terminated worker
|
||||
if (event.target !== this.worker) {
|
||||
return
|
||||
}
|
||||
|
||||
const response = event.data
|
||||
|
||||
switch (response.type) {
|
||||
case 'listening':
|
||||
this.listening = true
|
||||
for (const message of this.pendingMessages) {
|
||||
this.worker.postMessage(message)
|
||||
}
|
||||
this.pendingMessages.length = 0
|
||||
return
|
||||
|
||||
case 'loaded':
|
||||
this.lifecycleCallback?.({ type: 'loaded' })
|
||||
return
|
||||
|
||||
case 'loading-failed':
|
||||
this.loadingError = response.error
|
||||
this.pendingMessages.length = 0
|
||||
this.lifecycleCallback?.({
|
||||
type: 'loading-failed',
|
||||
error: response.error,
|
||||
})
|
||||
return
|
||||
|
||||
case 'output-line':
|
||||
this.outputCallback?.(
|
||||
response.stream,
|
||||
response.line,
|
||||
response.fileId,
|
||||
response.executionId
|
||||
)
|
||||
return
|
||||
|
||||
case 'run-code-result': {
|
||||
let success = response.success
|
||||
const failedUploads: string[] = []
|
||||
|
||||
if (success && response.outputFiles.length > 0) {
|
||||
const items: BatchUploadItem[] = response.outputFiles.map(file => ({
|
||||
file: new Blob([file.content as Uint8Array<ArrayBuffer>]),
|
||||
name: path.basename(file.relativePath),
|
||||
relativePath: file.relativePath,
|
||||
}))
|
||||
|
||||
try {
|
||||
const results = await this.fileUploader(items)
|
||||
for (const result of results) {
|
||||
if (result.status === 'error') {
|
||||
failedUploads.push(result.relativePath!)
|
||||
this.outputCallback?.(
|
||||
'stderr',
|
||||
`Failed to upload output file ${result.relativePath!}: ${result.error}`,
|
||||
response.fileId,
|
||||
response.executionId
|
||||
)
|
||||
}
|
||||
}
|
||||
if (failedUploads.length > 0) {
|
||||
success = false
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
this.outputCallback?.(
|
||||
'stderr',
|
||||
`Failed to upload output files: ${message}`,
|
||||
response.fileId,
|
||||
response.executionId
|
||||
)
|
||||
failedUploads.push(...items.map(item => item.relativePath!))
|
||||
success = false
|
||||
}
|
||||
}
|
||||
|
||||
const errorType =
|
||||
failedUploads.length > 0 ? 'UploadFileError' : response.errorType
|
||||
|
||||
this.lifecycleCallback?.({
|
||||
type: 'run-finished',
|
||||
fileId: response.fileId,
|
||||
executionId: response.executionId,
|
||||
success,
|
||||
outputs: response.outputs,
|
||||
failedUploads,
|
||||
imports: response.imports,
|
||||
errorType,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
-72
@@ -1,72 +0,0 @@
|
||||
export type OutputStream = 'stdout' | 'stderr' | 'info'
|
||||
|
||||
export type ProjectFileData = {
|
||||
relativePath: string
|
||||
content: string
|
||||
}
|
||||
|
||||
export type OutputFileData = {
|
||||
relativePath: string
|
||||
content: Uint8Array
|
||||
}
|
||||
|
||||
// Main thread -> Worker messages
|
||||
|
||||
export type InitRequest = {
|
||||
type: 'init'
|
||||
baseAssetPath: string
|
||||
}
|
||||
|
||||
export type RunCodeRequest = {
|
||||
type: 'run-code'
|
||||
fileId: string
|
||||
executionId: string
|
||||
code: string
|
||||
files: ProjectFileData[]
|
||||
}
|
||||
|
||||
export type PyodideWorkerRequest = InitRequest | RunCodeRequest
|
||||
|
||||
// Worker -> Main thread lifecycle and streaming events
|
||||
|
||||
export type ListeningEvent = { type: 'listening' }
|
||||
export type LoadedEvent = { type: 'loaded' }
|
||||
export type LoadingFailedEvent = { type: 'loading-failed'; error: string }
|
||||
|
||||
export type OutputLineEvent = {
|
||||
type: 'output-line'
|
||||
stream: OutputStream
|
||||
line: string
|
||||
fileId: string
|
||||
executionId: string
|
||||
}
|
||||
|
||||
export type PyodideWorkerEvent =
|
||||
| ListeningEvent
|
||||
| LoadedEvent
|
||||
| LoadingFailedEvent
|
||||
| OutputLineEvent
|
||||
|
||||
// Worker -> Main thread ID responses
|
||||
|
||||
export type ExecutionErrorType =
|
||||
| 'SyntaxError'
|
||||
| 'ModuleNotFoundError'
|
||||
| 'OutputLimitExceeded'
|
||||
| 'UploadFileError'
|
||||
| 'generic'
|
||||
|
||||
export type ExecutionResult = 'success' | 'error'
|
||||
|
||||
export type RunCodeResult = {
|
||||
type: 'run-code-result'
|
||||
fileId: string
|
||||
executionId: string
|
||||
success: boolean
|
||||
outputs: string[]
|
||||
outputFiles: OutputFileData[]
|
||||
imports: string[]
|
||||
errorType?: ExecutionErrorType
|
||||
}
|
||||
|
||||
export type PyodideWorkerResponse = PyodideWorkerEvent | RunCodeResult
|
||||
-53
@@ -1,53 +0,0 @@
|
||||
const BYTES_PER_MB = 1024 * 1024
|
||||
|
||||
export const MAX_OUTPUT_FILES = 50
|
||||
export const MAX_OUTPUT_TOTAL_BYTES = 100 * BYTES_PER_MB
|
||||
export const MAX_OUTPUT_FILE_BYTES = 50 * BYTES_PER_MB
|
||||
|
||||
export type OutputLimitViolation = {
|
||||
kind: 'count' | 'total-output-size' | 'single-file-size'
|
||||
message: string
|
||||
}
|
||||
|
||||
export function checkOutputCount(count: number): OutputLimitViolation | null {
|
||||
if (count > MAX_OUTPUT_FILES) {
|
||||
return {
|
||||
kind: 'count',
|
||||
message: `Output limit exceeded: ${count} files generated (max ${MAX_OUTPUT_FILES})`,
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export function checkOutputLimits(
|
||||
files: { path: string; size: number }[]
|
||||
): OutputLimitViolation | null {
|
||||
const countViolation = checkOutputCount(files.length)
|
||||
if (countViolation) {
|
||||
return countViolation
|
||||
}
|
||||
|
||||
let totalBytes = 0
|
||||
for (const file of files) {
|
||||
if (file.size > MAX_OUTPUT_FILE_BYTES) {
|
||||
const fileMB = Math.ceil(file.size / BYTES_PER_MB)
|
||||
const maxMB = MAX_OUTPUT_FILE_BYTES / BYTES_PER_MB
|
||||
return {
|
||||
kind: 'single-file-size',
|
||||
message: `Output limit exceeded: ${file.path} is ${fileMB}MB (max ${maxMB}MB per file)`,
|
||||
}
|
||||
}
|
||||
totalBytes += file.size
|
||||
}
|
||||
|
||||
if (totalBytes > MAX_OUTPUT_TOTAL_BYTES) {
|
||||
const totalMB = Math.ceil(totalBytes / BYTES_PER_MB)
|
||||
const maxMB = MAX_OUTPUT_TOTAL_BYTES / BYTES_PER_MB
|
||||
return {
|
||||
kind: 'total-output-size',
|
||||
message: `Output limit exceeded: ${totalMB}MB total (max ${maxMB}MB)`,
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
-309
@@ -1,309 +0,0 @@
|
||||
/// <reference lib="webworker" />
|
||||
import path from 'path-browserify'
|
||||
import type { PyodideInterface } from 'pyodide'
|
||||
import type {
|
||||
ExecutionErrorType,
|
||||
OutputFileData,
|
||||
InitRequest,
|
||||
ProjectFileData,
|
||||
PyodideWorkerRequest,
|
||||
RunCodeRequest,
|
||||
} from './pyodide-worker-messages'
|
||||
import {
|
||||
checkOutputCount,
|
||||
checkOutputLimits,
|
||||
} from './pyodide-worker-output-limits'
|
||||
|
||||
type PyodideFS = PyodideInterface['FS']
|
||||
type PyodideModule = typeof import('pyodide')
|
||||
|
||||
const PROJECT_FS_ROOT = '/project'
|
||||
const PROJECT_FS_PREFIX = `${PROJECT_FS_ROOT}/`
|
||||
const PYODIDE_INDEX_PATH = 'js/libs/pyodide/'
|
||||
|
||||
function classifyErrorType(errorMessage: string): ExecutionErrorType {
|
||||
if (errorMessage.includes('ModuleNotFoundError')) {
|
||||
return 'ModuleNotFoundError'
|
||||
}
|
||||
if (errorMessage.includes('SyntaxError')) {
|
||||
return 'SyntaxError'
|
||||
}
|
||||
return 'generic'
|
||||
}
|
||||
|
||||
function moduleNotFoundHelpMessage(): string {
|
||||
return (
|
||||
"Note: Only Pyodide's built-in packages are available in the browser. " +
|
||||
'Packages installed via pip cannot be used here. ' +
|
||||
'See https://pyodide.org/en/stable/usage/packages-in-pyodide.html ' +
|
||||
'for the list of supported packages.'
|
||||
)
|
||||
}
|
||||
|
||||
function ensureDirectoryExists(fs: PyodideFS, filePath: string) {
|
||||
const directory = path.dirname(filePath)
|
||||
if (directory === '.' || directory === '/') {
|
||||
return
|
||||
}
|
||||
|
||||
let currentPath = directory.startsWith('/') ? '/' : ''
|
||||
for (const part of directory.split('/').filter(Boolean)) {
|
||||
currentPath = path.posix.join(currentPath, part)
|
||||
try {
|
||||
const analysis = fs.analyzePath(currentPath)
|
||||
if (!analysis.exists) {
|
||||
fs.mkdir(currentPath)
|
||||
}
|
||||
} catch {
|
||||
// Ignore failures when a directory already exists.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function syncProjectFiles(fs: PyodideFS, files: ProjectFileData[]) {
|
||||
for (const file of files) {
|
||||
const runtimePath = path.posix.join(
|
||||
PROJECT_FS_ROOT,
|
||||
path.posix.normalize(file.relativePath)
|
||||
)
|
||||
ensureDirectoryExists(fs, runtimePath)
|
||||
fs.writeFile(runtimePath, file.content)
|
||||
}
|
||||
|
||||
fs.chdir(PROJECT_FS_ROOT)
|
||||
}
|
||||
|
||||
let pyodideModule: PyodideModule | null = null
|
||||
let pyodideIndexUrl: string | undefined
|
||||
|
||||
async function loadPyodideModule(pyodideIndexUrl: string) {
|
||||
const runtimeModuleUrl = `${pyodideIndexUrl}pyodide.mjs`
|
||||
|
||||
try {
|
||||
return (await import(
|
||||
/* webpackIgnore: true */ runtimeModuleUrl
|
||||
)) as PyodideModule
|
||||
} catch (loadError) {
|
||||
const loadErrorMessage =
|
||||
loadError instanceof Error ? loadError.message : String(loadError)
|
||||
throw new Error(
|
||||
`Unable to load Pyodide module from ${runtimeModuleUrl}. Original error: ${loadErrorMessage}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function handleInit(msg: InitRequest) {
|
||||
pyodideIndexUrl = new URL(PYODIDE_INDEX_PATH, msg.baseAssetPath).toString()
|
||||
|
||||
try {
|
||||
pyodideModule = await loadPyodideModule(pyodideIndexUrl)
|
||||
self.postMessage({ type: 'loaded' })
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error)
|
||||
console.error('Pyodide initialization failed', error)
|
||||
self.postMessage({
|
||||
type: 'loading-failed',
|
||||
error: errorMessage,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRunCode(msg: RunCodeRequest) {
|
||||
const { fileId, executionId } = msg
|
||||
|
||||
const writtenPaths = new Set<string>()
|
||||
const readPaths = new Set<string>()
|
||||
|
||||
const computeImports = () =>
|
||||
[...readPaths].filter(path => !writtenPaths.has(path))
|
||||
|
||||
const postFailure = (
|
||||
stream: 'stderr' | 'info',
|
||||
line: string,
|
||||
errorType: ExecutionErrorType = 'generic'
|
||||
) => {
|
||||
self.postMessage({
|
||||
type: 'output-line',
|
||||
stream,
|
||||
line,
|
||||
fileId,
|
||||
executionId,
|
||||
})
|
||||
self.postMessage({
|
||||
type: 'run-code-result',
|
||||
fileId,
|
||||
executionId,
|
||||
success: false,
|
||||
outputs: [],
|
||||
outputFiles: [],
|
||||
imports: computeImports(),
|
||||
errorType,
|
||||
})
|
||||
}
|
||||
|
||||
if (!pyodideModule || !pyodideIndexUrl) {
|
||||
postFailure('stderr', 'Pyodide is not initialized')
|
||||
return
|
||||
}
|
||||
|
||||
const instance = await pyodideModule.loadPyodide({
|
||||
env: { MPLBACKEND: 'Agg' },
|
||||
packageBaseUrl: `${pyodideIndexUrl}${pyodideModule.version}/`,
|
||||
})
|
||||
|
||||
instance.setStdout({
|
||||
batched: (line: string) => {
|
||||
self.postMessage({
|
||||
type: 'output-line',
|
||||
stream: 'stdout',
|
||||
line,
|
||||
fileId,
|
||||
executionId,
|
||||
})
|
||||
},
|
||||
})
|
||||
instance.setStderr({
|
||||
batched: (line: string) => {
|
||||
self.postMessage({
|
||||
type: 'output-line',
|
||||
stream: 'stderr',
|
||||
line,
|
||||
fileId,
|
||||
executionId,
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
const fs = instance.FS
|
||||
const originalWrite = fs.write as PyodideFS['write']
|
||||
const originalRead = fs.read as PyodideFS['read']
|
||||
let runError: unknown = null
|
||||
try {
|
||||
if (msg.files.length > 0) {
|
||||
syncProjectFiles(fs, msg.files)
|
||||
}
|
||||
|
||||
fs.write = ((...args: Parameters<PyodideFS['write']>) => {
|
||||
const [stream] = args
|
||||
// Only surface writes to the synced project workspace, not Pyodide internals.
|
||||
if (
|
||||
typeof stream?.path === 'string' &&
|
||||
stream.path.startsWith(PROJECT_FS_PREFIX)
|
||||
) {
|
||||
writtenPaths.add(stream.path)
|
||||
}
|
||||
|
||||
return originalWrite.call(fs, ...args)
|
||||
}) as PyodideFS['write']
|
||||
|
||||
fs.read = ((...args: Parameters<PyodideFS['read']>) => {
|
||||
const [stream] = args
|
||||
if (
|
||||
typeof stream?.path === 'string' &&
|
||||
stream.path.startsWith(PROJECT_FS_PREFIX)
|
||||
) {
|
||||
readPaths.add(stream.path)
|
||||
}
|
||||
|
||||
return originalRead.call(fs, ...args)
|
||||
}) as PyodideFS['read']
|
||||
|
||||
await instance.loadPackagesFromImports(msg.code)
|
||||
const result = await instance.runPythonAsync(msg.code)
|
||||
if (result !== undefined) {
|
||||
self.postMessage({
|
||||
type: 'output-line',
|
||||
stream: 'stdout',
|
||||
line: String(result),
|
||||
fileId,
|
||||
executionId,
|
||||
})
|
||||
}
|
||||
} catch (e) {
|
||||
runError = e
|
||||
}
|
||||
fs.write = originalWrite
|
||||
fs.read = originalRead
|
||||
|
||||
const paths = [...writtenPaths]
|
||||
|
||||
if (runError) {
|
||||
const errorMessage =
|
||||
runError instanceof Error ? runError.message : String(runError)
|
||||
const errorType = classifyErrorType(errorMessage)
|
||||
const fullMessage =
|
||||
errorType === 'ModuleNotFoundError'
|
||||
? `${errorMessage}\n${moduleNotFoundHelpMessage()}`
|
||||
: errorMessage
|
||||
postFailure('stderr', fullMessage, errorType)
|
||||
return
|
||||
}
|
||||
|
||||
const countViolation = checkOutputCount(paths.length)
|
||||
if (countViolation) {
|
||||
postFailure('info', countViolation.message, 'OutputLimitExceeded')
|
||||
return
|
||||
}
|
||||
|
||||
const filesWithSizes: { path: string; size: number }[] = []
|
||||
for (const writtenPath of paths) {
|
||||
try {
|
||||
filesWithSizes.push({
|
||||
path: writtenPath,
|
||||
size: fs.stat(writtenPath).size,
|
||||
})
|
||||
} catch {
|
||||
// A script can write a file and later delete or rename it before the run
|
||||
// finishes; fs.stat would then throw and we'd never post a
|
||||
// run-code-result, leaving the UI stuck. Skip paths we can't stat.
|
||||
}
|
||||
}
|
||||
|
||||
const sizeViolation = checkOutputLimits(filesWithSizes)
|
||||
if (sizeViolation) {
|
||||
postFailure('info', sizeViolation.message, 'OutputLimitExceeded')
|
||||
return
|
||||
}
|
||||
|
||||
const outputFiles: OutputFileData[] = []
|
||||
const transferables: Transferable[] = []
|
||||
for (const { path: writtenPath } of filesWithSizes) {
|
||||
const content = fs.readFile(writtenPath)
|
||||
const relativePath = writtenPath.slice(PROJECT_FS_PREFIX.length)
|
||||
outputFiles.push({ relativePath, content })
|
||||
if (content.buffer instanceof ArrayBuffer) {
|
||||
transferables.push(content.buffer)
|
||||
}
|
||||
}
|
||||
|
||||
// The transferables moves ownership of each ArrayBuffer to the main thread
|
||||
// instead of structured-cloning it. The buffers are already referenced from
|
||||
// outputFiles.content; listing them here just swaps copy for move, so file
|
||||
// contents travel through once rather than being allocated on both sides.
|
||||
self.postMessage(
|
||||
{
|
||||
type: 'run-code-result',
|
||||
fileId,
|
||||
executionId,
|
||||
success: true,
|
||||
outputs: filesWithSizes.map(f => f.path),
|
||||
outputFiles,
|
||||
imports: computeImports(),
|
||||
},
|
||||
transferables
|
||||
)
|
||||
}
|
||||
|
||||
self.addEventListener('message', async event => {
|
||||
const msg = event.data as PyodideWorkerRequest
|
||||
switch (msg.type) {
|
||||
case 'init':
|
||||
await handleInit(msg)
|
||||
break
|
||||
case 'run-code':
|
||||
await handleRunCode(msg)
|
||||
break
|
||||
}
|
||||
})
|
||||
|
||||
self.postMessage({ type: 'listening' })
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
import { useEffect, useMemo, useSyncExternalStore } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import classNames from 'classnames'
|
||||
import OLButton from '@/shared/components/ol/ol-button'
|
||||
import OLButtonToolbar from '@/shared/components/ol/ol-button-toolbar'
|
||||
import MaterialIcon from '@/shared/components/material-icon'
|
||||
import SplitTestBadge from '@/shared/components/split-test-badge'
|
||||
import { sendMB } from '@/infrastructure/event-tracking'
|
||||
import { useEditorOpenDocContext } from '@/features/ide-react/context/editor-open-doc-context'
|
||||
import { usePythonExecutionContext } from '@/features/ide-react/context/python-execution-context'
|
||||
import { DEFAULT_STATE } from './python-runner'
|
||||
|
||||
const emptySubscribe = () => () => {}
|
||||
const getDefaultState = () => DEFAULT_STATE
|
||||
|
||||
export default function PythonOutputPane() {
|
||||
const { t } = useTranslation()
|
||||
const { currentDocumentId, openDocName } = useEditorOpenDocContext()
|
||||
const { getPythonRunner } = usePythonExecutionContext()
|
||||
const pythonRunner = useMemo(
|
||||
() => (currentDocumentId ? getPythonRunner(currentDocumentId) : null),
|
||||
[currentDocumentId, getPythonRunner]
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
if (!currentDocumentId || !openDocName) {
|
||||
return
|
||||
}
|
||||
sendMB('script-runner-opened', {
|
||||
fileName: openDocName,
|
||||
fileExtension: 'py',
|
||||
editorMode: 'code',
|
||||
})
|
||||
}, [currentDocumentId, openDocName])
|
||||
|
||||
const { output, error, status } = useSyncExternalStore(
|
||||
pythonRunner ? pythonRunner.subscribe : emptySubscribe,
|
||||
pythonRunner ? pythonRunner.getState : getDefaultState
|
||||
)
|
||||
|
||||
if (!pythonRunner) {
|
||||
return null
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="ide-redesign-python-output-pane">
|
||||
<OLButtonToolbar className="ide-redesign-python-output-pane-toolbar">
|
||||
<div className="ide-redesign-python-output-pane-toolbar-left">
|
||||
<div
|
||||
className={classNames(
|
||||
'ide-redesign-python-output-pane-run-button-wrapper',
|
||||
{
|
||||
'compile-button-group-running': status === 'running',
|
||||
}
|
||||
)}
|
||||
>
|
||||
<OLButton
|
||||
onClick={() => {
|
||||
if (status === 'running') {
|
||||
pythonRunner.interrupt()
|
||||
} else {
|
||||
pythonRunner.run()
|
||||
}
|
||||
}}
|
||||
variant={status === 'running' ? 'danger' : 'primary'}
|
||||
className="align-items-center py-0 px-3"
|
||||
disabled={status === 'loading'}
|
||||
aria-label={
|
||||
status === 'running'
|
||||
? t('stop_python_execution')
|
||||
: t('run_python_code')
|
||||
}
|
||||
>
|
||||
{status === 'running' ? t('stop') : t('run')}
|
||||
<MaterialIcon
|
||||
type={status === 'running' ? 'stop' : 'play_arrow'}
|
||||
className="ml-2"
|
||||
/>
|
||||
</OLButton>
|
||||
</div>
|
||||
<SplitTestBadge
|
||||
splitTestName="overleaf-code"
|
||||
displayOnVariants={['enabled']}
|
||||
/>
|
||||
</div>
|
||||
</OLButtonToolbar>
|
||||
|
||||
<div className="ide-redesign-python-output-pane-body">
|
||||
{status === 'loading' && (
|
||||
<div className="ide-redesign-python-output-pane-placeholder">
|
||||
{t('loading_python_runtime')}
|
||||
</div>
|
||||
)}
|
||||
{status !== 'loading' && !error && output.length === 0 && (
|
||||
<div className="ide-redesign-python-output-pane-placeholder">
|
||||
{t('run_current_script_to_see_output')}
|
||||
</div>
|
||||
)}
|
||||
{error && (
|
||||
<div className="ide-redesign-python-output-pane-error">{error}</div>
|
||||
)}
|
||||
{output.map((entry, index) => (
|
||||
<div
|
||||
className={`ide-redesign-python-output-pane-line ide-redesign-python-output-pane-line-${entry.stream}`}
|
||||
key={index}
|
||||
>
|
||||
{entry.line}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
import { GlobalToastGeneratorEntry } from '@/features/ide-react/components/global-toasts'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
const stripProjectPrefix = (path: string) => path.replace(/^\/project\/?/, '')
|
||||
|
||||
const PythonFilesSavedToast = ({ paths }: { paths: string[] }) => {
|
||||
const { t } = useTranslation()
|
||||
if (paths.length === 1) {
|
||||
return (
|
||||
<span>
|
||||
{t('x_saved_to_your_project', {
|
||||
fileName: stripProjectPrefix(paths[0]),
|
||||
})}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
return (
|
||||
<span>{t('x_files_saved_to_your_project', { count: paths.length })}</span>
|
||||
)
|
||||
}
|
||||
|
||||
const isStringArray = (value: unknown): value is string[] =>
|
||||
Array.isArray(value) && value.every(v => typeof v === 'string')
|
||||
|
||||
const generators: GlobalToastGeneratorEntry[] = [
|
||||
{
|
||||
key: 'python:files-saved',
|
||||
generator: ({ paths }) => ({
|
||||
content: (
|
||||
<PythonFilesSavedToast paths={isStringArray(paths) ? paths : []} />
|
||||
),
|
||||
type: 'success',
|
||||
autoHide: true,
|
||||
delay: 5000,
|
||||
isDismissible: true,
|
||||
}),
|
||||
},
|
||||
]
|
||||
|
||||
export default generators
|
||||
|
||||
export const showPythonFilesSavedToast = (paths: string[]) => {
|
||||
window.dispatchEvent(
|
||||
new CustomEvent('ide:show-toast', {
|
||||
detail: { key: 'python:files-saved', paths },
|
||||
})
|
||||
)
|
||||
}
|
||||
-305
@@ -1,305 +0,0 @@
|
||||
// Per-file Python execution manager. Each PythonRunner owns a PyodideWorkerClient
|
||||
// and exposes a subscribe/getState API for use with useSyncExternalStore,
|
||||
// so React components can reactively read execution status and output.
|
||||
import path from 'path-browserify'
|
||||
import { v4 as uuid } from 'uuid'
|
||||
import { debugConsole } from '@/utils/debugging'
|
||||
import { sendMB } from '@/infrastructure/event-tracking'
|
||||
import { PyodideWorkerClient } from './pyodide-worker-client'
|
||||
import { showPythonFilesSavedToast } from './python-output-toasts'
|
||||
import type { OutputStream } from './pyodide-worker-messages'
|
||||
import type {
|
||||
BatchUploadItem,
|
||||
UploadResult,
|
||||
} from '@/infrastructure/batch-file-uploader'
|
||||
|
||||
export type FileUploader = (items: BatchUploadItem[]) => Promise<UploadResult[]>
|
||||
|
||||
const MAX_OUTPUT_LINES = 100
|
||||
const PROJECT_FS_PREFIX = '/project/'
|
||||
|
||||
function stripProjectFsPrefix(p: string): string {
|
||||
return p.startsWith(PROJECT_FS_PREFIX) ? p.slice(PROJECT_FS_PREFIX.length) : p
|
||||
}
|
||||
|
||||
export type ExecutionStatus =
|
||||
| 'loading'
|
||||
| 'idle'
|
||||
| 'running'
|
||||
| 'finished'
|
||||
| 'errored'
|
||||
|
||||
export type ExecutionContext = {
|
||||
code: string
|
||||
files: { relativePath: string; content: string }[]
|
||||
}
|
||||
|
||||
type Listener = () => void
|
||||
|
||||
export type OutputLine = {
|
||||
stream: OutputStream
|
||||
line: string
|
||||
}
|
||||
|
||||
export type PythonRunnerState = {
|
||||
output: OutputLine[]
|
||||
status: ExecutionStatus
|
||||
error: string | null
|
||||
}
|
||||
|
||||
export const DEFAULT_STATE: PythonRunnerState = {
|
||||
output: [],
|
||||
status: 'loading',
|
||||
error: null,
|
||||
}
|
||||
|
||||
export class PythonRunner {
|
||||
readonly fileId: string
|
||||
private client: PyodideWorkerClient | null = null
|
||||
private readonly baseAssetPath: string
|
||||
private readonly createWorker: () => Worker
|
||||
private readonly getExecutionContext: () => Promise<ExecutionContext | null>
|
||||
private readonly fileUploader: FileUploader
|
||||
|
||||
private listeners = new Set<Listener>()
|
||||
|
||||
private activeExecution: { id: string; startedAt: number } | null = null
|
||||
private state: PythonRunnerState = DEFAULT_STATE
|
||||
|
||||
constructor(
|
||||
fileId: string,
|
||||
baseAssetPath: string,
|
||||
getExecutionContext: () => Promise<ExecutionContext | null>,
|
||||
createWorker: () => Worker,
|
||||
fileUploader: FileUploader
|
||||
) {
|
||||
this.fileId = fileId
|
||||
this.baseAssetPath = baseAssetPath
|
||||
this.createWorker = createWorker
|
||||
this.getExecutionContext = getExecutionContext
|
||||
this.fileUploader = fileUploader
|
||||
}
|
||||
|
||||
subscribe = (listener: Listener): (() => void) => {
|
||||
this.listeners.add(listener)
|
||||
return () => this.listeners.delete(listener)
|
||||
}
|
||||
|
||||
getState = () => this.state
|
||||
|
||||
private updateState(fields: Partial<PythonRunnerState>) {
|
||||
const prev = this.state
|
||||
const output = fields.output ?? prev.output
|
||||
const status = fields.status ?? prev.status
|
||||
const error = fields.error !== undefined ? fields.error : prev.error
|
||||
|
||||
if (
|
||||
output === prev.output &&
|
||||
status === prev.status &&
|
||||
error === prev.error
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
this.state = { output, status, error }
|
||||
|
||||
for (const listener of this.listeners) {
|
||||
listener()
|
||||
}
|
||||
}
|
||||
|
||||
init() {
|
||||
if (this.client) {
|
||||
return
|
||||
}
|
||||
|
||||
this.updateState({ status: 'loading', error: null })
|
||||
|
||||
this.client = new PyodideWorkerClient({
|
||||
baseAssetPath: this.baseAssetPath,
|
||||
createWorker: this.createWorker,
|
||||
fileUploader: this.fileUploader,
|
||||
onLifecycle: event => {
|
||||
switch (event.type) {
|
||||
case 'loaded':
|
||||
this.updateState({ status: 'idle', error: null })
|
||||
return
|
||||
|
||||
case 'loading-failed':
|
||||
debugConsole.error('Failed to load Python runtime', event.error)
|
||||
this.updateState({ status: 'errored', error: event.error })
|
||||
return
|
||||
|
||||
case 'run-finished': {
|
||||
const active = this.activeExecution
|
||||
if (
|
||||
event.fileId !== this.fileId ||
|
||||
active?.id !== event.executionId
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
this.activeExecution = null
|
||||
|
||||
sendMB('script-runner-execution-completed', {
|
||||
result: event.success ? 'success' : 'error',
|
||||
errorType: event.success ? undefined : event.errorType,
|
||||
executionTimeMs: Math.round(performance.now() - active.startedAt),
|
||||
filesImportedCount: event.imports.length,
|
||||
filesImportedExtensions: collectExtensions(event.imports),
|
||||
filesWrittenCount: event.outputs.length,
|
||||
filesWrittenExtensions: collectExtensions(event.outputs),
|
||||
})
|
||||
|
||||
// event.outputs are full worker paths (/project/foo.txt) while
|
||||
// event.failedUploads are relativePaths (foo.txt); strip the
|
||||
// prefix before comparing.
|
||||
const failed = new Set(event.failedUploads)
|
||||
const uploadedPaths = event.outputs
|
||||
.map(stripProjectFsPrefix)
|
||||
.filter(p => !failed.has(p))
|
||||
if (uploadedPaths.length > 0) {
|
||||
showPythonFilesSavedToast(uploadedPaths)
|
||||
}
|
||||
|
||||
this.updateState({ status: 'finished' })
|
||||
}
|
||||
}
|
||||
},
|
||||
onOutput: (stream, line, fileId, executionId) => {
|
||||
if (
|
||||
fileId !== this.fileId ||
|
||||
this.activeExecution?.id !== executionId
|
||||
) {
|
||||
return
|
||||
}
|
||||
this.updateState({
|
||||
output: appendCapped(this.state.output, { stream, line }),
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async run() {
|
||||
if (!this.client || this.state.status === 'loading') {
|
||||
return
|
||||
}
|
||||
|
||||
let context: ExecutionContext | null
|
||||
try {
|
||||
context = await this.getExecutionContext()
|
||||
} catch (err) {
|
||||
debugConsole.error('Failed to build execution context', err)
|
||||
this.updateState({ status: 'errored', error: formatError(err) })
|
||||
return
|
||||
}
|
||||
|
||||
// Re-check after await — status may have changed but TypeScript
|
||||
// still narrows from the pre-await check, so we cast back.
|
||||
if (
|
||||
!context ||
|
||||
!this.client ||
|
||||
(this.state.status as ExecutionStatus) === 'loading'
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
const { code, files } = context
|
||||
|
||||
sendMB('script-runner-run-clicked', {
|
||||
scriptLineCount: countLines(code),
|
||||
})
|
||||
|
||||
const executionId = uuid()
|
||||
this.activeExecution = { id: executionId, startedAt: performance.now() }
|
||||
this.updateState({ status: 'running', output: [], error: null })
|
||||
|
||||
try {
|
||||
this.client.runCode(code, {
|
||||
fileId: this.fileId,
|
||||
executionId,
|
||||
files,
|
||||
})
|
||||
} catch (runError) {
|
||||
if (this.activeExecution?.id !== executionId) {
|
||||
return
|
||||
}
|
||||
this.activeExecution = null
|
||||
this.updateState({ status: 'errored', error: formatError(runError) })
|
||||
}
|
||||
}
|
||||
|
||||
interrupt() {
|
||||
if (!this.client) {
|
||||
return
|
||||
}
|
||||
|
||||
if (this.state.status === 'running' && this.activeExecution) {
|
||||
sendMB('script-runner-stop-clicked', {
|
||||
timeBeforeStopMs: Math.round(
|
||||
performance.now() - this.activeExecution.startedAt
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
this.client.reset()
|
||||
this.activeExecution = null
|
||||
|
||||
// The worker is terminated and recreated by reset(), so it needs to
|
||||
// reload Pyodide. The 'loaded' lifecycle callback will transition
|
||||
// back to 'idle'.
|
||||
this.updateState({
|
||||
status: 'loading',
|
||||
output:
|
||||
this.state.status === 'running'
|
||||
? appendCapped(this.state.output, {
|
||||
stream: 'info',
|
||||
line: 'Execution interrupted',
|
||||
})
|
||||
: this.state.output,
|
||||
})
|
||||
}
|
||||
|
||||
destroy() {
|
||||
if (this.client) {
|
||||
this.client.destroy()
|
||||
this.client = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function appendCapped(existing: OutputLine[], entry: OutputLine): OutputLine[] {
|
||||
const updated = [...existing, entry]
|
||||
return updated.length > MAX_OUTPUT_LINES
|
||||
? updated.slice(-MAX_OUTPUT_LINES)
|
||||
: updated
|
||||
}
|
||||
|
||||
function formatError(error: unknown): string {
|
||||
if (error instanceof Error) {
|
||||
return error.message
|
||||
}
|
||||
return String(error)
|
||||
}
|
||||
|
||||
function countLines(code: string): number {
|
||||
if (code.length === 0) {
|
||||
return 0
|
||||
}
|
||||
return code.split('\n').length
|
||||
}
|
||||
|
||||
function extractExtension(filePath: string): string {
|
||||
return path.extname(filePath).slice(1).toLowerCase()
|
||||
}
|
||||
|
||||
function collectExtensions(filePaths: string[]): string {
|
||||
const seen = new Set<string>()
|
||||
for (const filePath of filePaths) {
|
||||
const ext = extractExtension(filePath)
|
||||
if (ext) {
|
||||
seen.add(ext)
|
||||
}
|
||||
}
|
||||
return Array.from(seen).join(',')
|
||||
}
|
||||
Reference in New Issue
Block a user