From 7601071d1818fec809595cda8aa46f6d185d1139 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Sun, 27 Sep 2026 04:13:58 +0200 Subject: [PATCH] Python script runner: initial commit - restore upstream code removed by commit b90a850 - migrate restored code to modules - improve Run / Stop button design --- server-ce/genScript.js | 2 + services/web/config/settings.defaults.js | 12 +- .../editor/python/pyodide-worker-client.ts | 225 +++++ .../editor/python/pyodide-worker-messages.ts | 72 ++ .../python/pyodide-worker-output-limits.ts | 53 ++ .../editor/python/pyodide.worker.ts | 309 +++++++ .../editor/python/python-output-pane.tsx | 97 +++ .../editor/python/python-output-toasts.tsx | 48 ++ .../components/editor/python/python-runner.ts | 305 +++++++ .../components/layout/python-editor-split.tsx | 31 + .../js/context/python-execution-context.tsx | 167 ++++ .../frontend/stylesheets/python-runner.scss | 71 ++ services/web/modules/python-runner/index.mjs | 6 + .../components/python-output-pane.spec.tsx | 481 +++++++++++ .../unit/editor/pyodide-worker-client.spec.ts | 778 ++++++++++++++++++ .../pyodide-worker-output-limits.spec.ts | 93 +++ .../test/unit/editor/python-runner.spec.ts | 499 +++++++++++ .../test/unit/editor/worker-mock.ts | 35 + .../web/scripts/fetch-pyodide-packages.mjs | 136 +++ 19 files changed, 3419 insertions(+), 1 deletion(-) create mode 100644 services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-client.ts create mode 100644 services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-messages.ts create mode 100644 services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-output-limits.ts create mode 100644 services/web/modules/python-runner/frontend/js/components/editor/python/pyodide.worker.ts create mode 100644 services/web/modules/python-runner/frontend/js/components/editor/python/python-output-pane.tsx create mode 100644 services/web/modules/python-runner/frontend/js/components/editor/python/python-output-toasts.tsx create mode 100644 services/web/modules/python-runner/frontend/js/components/editor/python/python-runner.ts create mode 100644 services/web/modules/python-runner/frontend/js/components/layout/python-editor-split.tsx create mode 100644 services/web/modules/python-runner/frontend/js/context/python-execution-context.tsx create mode 100644 services/web/modules/python-runner/frontend/stylesheets/python-runner.scss create mode 100644 services/web/modules/python-runner/index.mjs create mode 100644 services/web/modules/python-runner/test/frontend/components/python-output-pane.spec.tsx create mode 100644 services/web/modules/python-runner/test/unit/editor/pyodide-worker-client.spec.ts create mode 100644 services/web/modules/python-runner/test/unit/editor/pyodide-worker-output-limits.spec.ts create mode 100644 services/web/modules/python-runner/test/unit/editor/python-runner.spec.ts create mode 100644 services/web/modules/python-runner/test/unit/editor/worker-mock.ts create mode 100644 services/web/scripts/fetch-pyodide-packages.mjs diff --git a/server-ce/genScript.js b/server-ce/genScript.js index a5513a0777..55b915fda0 100644 --- a/server-ce/genScript.js +++ b/server-ce/genScript.js @@ -23,6 +23,8 @@ switch (process.argv.pop()) { console.log('yarn install') // run webpack console.log('yarn run webpack:production') + // fetch Pyodide wheel bundle (numpy/matplotlib/etc.) for the python-runner module + console.log('yarn run pyodide:fetch') // uninstall webpack and frontend dependencies console.log('yarn workspaces focus --all --production') diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index ed73c1d81c..807c2c7e23 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -1063,7 +1063,12 @@ module.exports = { ), ], mainEditorLayoutPanels: [], - pythonRunner: [], + pythonRunner: [ + Path.resolve( + __dirname, + '../modules/python-runner/frontend/js/components/layout/python-editor-split' + ), + ], langFeedbackLinkingWidgets: [], labsExperiments: [], integrationLinkingWidgets: [ @@ -1129,6 +1134,10 @@ module.exports = { __dirname, '../frontend/js/features/pdf-preview/components/synctex-toasts' ), + Path.resolve( + __dirname, + '../modules/python-runner/frontend/js/components/editor/python/python-output-toasts' + ), ], editorSidebarComponents: [ Path.resolve( @@ -1239,6 +1248,7 @@ module.exports = { splitTestOverrides: { 'themed-modals': 'enabled', + 'overleaf-code': 'enabled', ...(process.env.ENABLE_EDITOR_TABS?.toLowerCase() === 'true' ? { 'editor-tabs': 'enabled', } : {}), diff --git a/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-client.ts b/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-client.ts new file mode 100644 index 0000000000..cf4fe16b3b --- /dev/null +++ b/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-client.ts @@ -0,0 +1,225 @@ +import path from 'path-browserify' +import type { + ExecutionErrorType, + OutputStream, + ProjectFileData, + PyodideWorkerRequest, + PyodideWorkerResponse, +} from './pyodide-worker-messages' +import type { BatchUploadItem } from '@/infrastructure/batch-file-uploader' +import type { FileUploader } from './python-runner' + +export type OutputCallback = ( + stream: OutputStream, + line: string, + fileId: string, + executionId: string +) => void + +export type LifecycleCallback = ( + event: + | { type: 'loaded' } + | { type: 'loading-failed'; error: string } + | { + type: 'run-finished' + fileId: string + executionId: string + success: boolean + outputs: string[] + failedUploads: string[] + imports: string[] + errorType?: ExecutionErrorType + } +) => void + +export class PyodideWorkerClient { + private worker: Worker + private baseAssetPath: string + private createWorker: () => Worker + private listening = false + private destroyed = false + private loadingError: string | null = null + private pendingMessages: PyodideWorkerRequest[] = [] + private outputCallback: OutputCallback | null + private lifecycleCallback: LifecycleCallback | null + private fileUploader: FileUploader + + constructor(options: { + baseAssetPath: string + createWorker: () => Worker + onOutput?: OutputCallback + onLifecycle?: LifecycleCallback + fileUploader: FileUploader + }) { + this.baseAssetPath = options.baseAssetPath + this.createWorker = options.createWorker + this.outputCallback = options.onOutput ?? null + this.lifecycleCallback = options.onLifecycle ?? null + this.fileUploader = options.fileUploader + this.worker = this.createWorker() + this.worker.addEventListener('message', this.receive) + + this.queueMessage({ + type: 'init', + baseAssetPath: this.baseAssetPath, + }) + } + + runCode( + code: string, + options: { fileId: string; executionId: string; files: ProjectFileData[] } + ): void { + if (this.destroyed) { + throw new Error('Pyodide worker client has been destroyed') + } + + if (this.loadingError) { + throw new Error(this.loadingError) + } + + this.queueMessage({ + type: 'run-code', + code, + fileId: options.fileId, + executionId: options.executionId, + files: options.files, + }) + } + + reset(): void { + if (this.destroyed) { + return + } + + // Terminate the current worker immediately + this.worker.terminate() + this.pendingMessages.length = 0 + + // Reset state for the new worker + this.listening = false + this.loadingError = null + + // Create a fresh worker and re-initialize Pyodide + this.worker = this.createWorker() + this.worker.addEventListener('message', this.receive) + this.queueMessage({ + type: 'init', + baseAssetPath: this.baseAssetPath, + }) + } + + destroy() { + if (this.destroyed) { + return + } + + this.destroyed = true + this.pendingMessages.length = 0 + + this.worker.terminate() + } + + private queueMessage(message: PyodideWorkerRequest) { + if (this.listening) { + this.worker.postMessage(message) + } else { + this.pendingMessages.push(message) + } + } + + private receive = async (event: MessageEvent) => { + // Discard messages from a previously terminated worker + if (event.target !== this.worker) { + return + } + + const response = event.data + + switch (response.type) { + case 'listening': + this.listening = true + for (const message of this.pendingMessages) { + this.worker.postMessage(message) + } + this.pendingMessages.length = 0 + return + + case 'loaded': + this.lifecycleCallback?.({ type: 'loaded' }) + return + + case 'loading-failed': + this.loadingError = response.error + this.pendingMessages.length = 0 + this.lifecycleCallback?.({ + type: 'loading-failed', + error: response.error, + }) + return + + case 'output-line': + this.outputCallback?.( + response.stream, + response.line, + response.fileId, + response.executionId + ) + return + + case 'run-code-result': { + let success = response.success + const failedUploads: string[] = [] + + if (success && response.outputFiles.length > 0) { + const items: BatchUploadItem[] = response.outputFiles.map(file => ({ + file: new Blob([file.content as Uint8Array]), + name: path.basename(file.relativePath), + relativePath: file.relativePath, + })) + + try { + const results = await this.fileUploader(items) + for (const result of results) { + if (result.status === 'error') { + failedUploads.push(result.relativePath!) + this.outputCallback?.( + 'stderr', + `Failed to upload output file ${result.relativePath!}: ${result.error}`, + response.fileId, + response.executionId + ) + } + } + if (failedUploads.length > 0) { + success = false + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + this.outputCallback?.( + 'stderr', + `Failed to upload output files: ${message}`, + response.fileId, + response.executionId + ) + failedUploads.push(...items.map(item => item.relativePath!)) + success = false + } + } + + const errorType = + failedUploads.length > 0 ? 'UploadFileError' : response.errorType + + this.lifecycleCallback?.({ + type: 'run-finished', + fileId: response.fileId, + executionId: response.executionId, + success, + outputs: response.outputs, + failedUploads, + imports: response.imports, + errorType, + }) + } + } + } +} diff --git a/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-messages.ts b/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-messages.ts new file mode 100644 index 0000000000..9b448a74c3 --- /dev/null +++ b/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-messages.ts @@ -0,0 +1,72 @@ +export type OutputStream = 'stdout' | 'stderr' | 'info' + +export type ProjectFileData = { + relativePath: string + content: string +} + +export type OutputFileData = { + relativePath: string + content: Uint8Array +} + +// Main thread -> Worker messages + +export type InitRequest = { + type: 'init' + baseAssetPath: string +} + +export type RunCodeRequest = { + type: 'run-code' + fileId: string + executionId: string + code: string + files: ProjectFileData[] +} + +export type PyodideWorkerRequest = InitRequest | RunCodeRequest + +// Worker -> Main thread lifecycle and streaming events + +export type ListeningEvent = { type: 'listening' } +export type LoadedEvent = { type: 'loaded' } +export type LoadingFailedEvent = { type: 'loading-failed'; error: string } + +export type OutputLineEvent = { + type: 'output-line' + stream: OutputStream + line: string + fileId: string + executionId: string +} + +export type PyodideWorkerEvent = + | ListeningEvent + | LoadedEvent + | LoadingFailedEvent + | OutputLineEvent + +// Worker -> Main thread ID responses + +export type ExecutionErrorType = + | 'SyntaxError' + | 'ModuleNotFoundError' + | 'OutputLimitExceeded' + | 'UploadFileError' + | 'generic' + +export type ExecutionResult = 'success' | 'error' + +export type RunCodeResult = { + type: 'run-code-result' + fileId: string + executionId: string + success: boolean + outputs: string[] + outputFiles: OutputFileData[] + imports: string[] + errorType?: ExecutionErrorType +} + +export type PyodideWorkerResponse = PyodideWorkerEvent | RunCodeResult diff --git a/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-output-limits.ts b/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-output-limits.ts new file mode 100644 index 0000000000..0897251e46 --- /dev/null +++ b/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide-worker-output-limits.ts @@ -0,0 +1,53 @@ +const BYTES_PER_MB = 1024 * 1024 + +export const MAX_OUTPUT_FILES = 50 +export const MAX_OUTPUT_TOTAL_BYTES = 100 * BYTES_PER_MB +export const MAX_OUTPUT_FILE_BYTES = 50 * BYTES_PER_MB + +export type OutputLimitViolation = { + kind: 'count' | 'total-output-size' | 'single-file-size' + message: string +} + +export function checkOutputCount(count: number): OutputLimitViolation | null { + if (count > MAX_OUTPUT_FILES) { + return { + kind: 'count', + message: `Output limit exceeded: ${count} files generated (max ${MAX_OUTPUT_FILES})`, + } + } + return null +} + +export function checkOutputLimits( + files: { path: string; size: number }[] +): OutputLimitViolation | null { + const countViolation = checkOutputCount(files.length) + if (countViolation) { + return countViolation + } + + let totalBytes = 0 + for (const file of files) { + if (file.size > MAX_OUTPUT_FILE_BYTES) { + const fileMB = Math.ceil(file.size / BYTES_PER_MB) + const maxMB = MAX_OUTPUT_FILE_BYTES / BYTES_PER_MB + return { + kind: 'single-file-size', + message: `Output limit exceeded: ${file.path} is ${fileMB}MB (max ${maxMB}MB per file)`, + } + } + totalBytes += file.size + } + + if (totalBytes > MAX_OUTPUT_TOTAL_BYTES) { + const totalMB = Math.ceil(totalBytes / BYTES_PER_MB) + const maxMB = MAX_OUTPUT_TOTAL_BYTES / BYTES_PER_MB + return { + kind: 'total-output-size', + message: `Output limit exceeded: ${totalMB}MB total (max ${maxMB}MB)`, + } + } + + return null +} diff --git a/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide.worker.ts b/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide.worker.ts new file mode 100644 index 0000000000..0c942b819e --- /dev/null +++ b/services/web/modules/python-runner/frontend/js/components/editor/python/pyodide.worker.ts @@ -0,0 +1,309 @@ +/// +import path from 'path-browserify' +import type { PyodideInterface } from 'pyodide' +import type { + ExecutionErrorType, + OutputFileData, + InitRequest, + ProjectFileData, + PyodideWorkerRequest, + RunCodeRequest, +} from './pyodide-worker-messages' +import { + checkOutputCount, + checkOutputLimits, +} from './pyodide-worker-output-limits' + +type PyodideFS = PyodideInterface['FS'] +type PyodideModule = typeof import('pyodide') + +const PROJECT_FS_ROOT = '/project' +const PROJECT_FS_PREFIX = `${PROJECT_FS_ROOT}/` +const PYODIDE_INDEX_PATH = 'js/libs/pyodide/' + +function classifyErrorType(errorMessage: string): ExecutionErrorType { + if (errorMessage.includes('ModuleNotFoundError')) { + return 'ModuleNotFoundError' + } + if (errorMessage.includes('SyntaxError')) { + return 'SyntaxError' + } + return 'generic' +} + +function moduleNotFoundHelpMessage(): string { + return ( + "Note: Only Pyodide's built-in packages are available in the browser. " + + 'Packages installed via pip cannot be used here. ' + + 'See https://pyodide.org/en/stable/usage/packages-in-pyodide.html ' + + 'for the list of supported packages.' + ) +} + +function ensureDirectoryExists(fs: PyodideFS, filePath: string) { + const directory = path.dirname(filePath) + if (directory === '.' || directory === '/') { + return + } + + let currentPath = directory.startsWith('/') ? '/' : '' + for (const part of directory.split('/').filter(Boolean)) { + currentPath = path.posix.join(currentPath, part) + try { + const analysis = fs.analyzePath(currentPath) + if (!analysis.exists) { + fs.mkdir(currentPath) + } + } catch { + // Ignore failures when a directory already exists. + } + } +} + +function syncProjectFiles(fs: PyodideFS, files: ProjectFileData[]) { + for (const file of files) { + const runtimePath = path.posix.join( + PROJECT_FS_ROOT, + path.posix.normalize(file.relativePath) + ) + ensureDirectoryExists(fs, runtimePath) + fs.writeFile(runtimePath, file.content) + } + + fs.chdir(PROJECT_FS_ROOT) +} + +let pyodideModule: PyodideModule | null = null +let pyodideIndexUrl: string | undefined + +async function loadPyodideModule(pyodideIndexUrl: string) { + const runtimeModuleUrl = `${pyodideIndexUrl}pyodide.mjs` + + try { + return (await import( + /* webpackIgnore: true */ runtimeModuleUrl + )) as PyodideModule + } catch (loadError) { + const loadErrorMessage = + loadError instanceof Error ? loadError.message : String(loadError) + throw new Error( + `Unable to load Pyodide module from ${runtimeModuleUrl}. Original error: ${loadErrorMessage}` + ) + } +} + +async function handleInit(msg: InitRequest) { + pyodideIndexUrl = new URL(PYODIDE_INDEX_PATH, msg.baseAssetPath).toString() + + try { + pyodideModule = await loadPyodideModule(pyodideIndexUrl) + self.postMessage({ type: 'loaded' }) + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error) + console.error('Pyodide initialization failed', error) + self.postMessage({ + type: 'loading-failed', + error: errorMessage, + }) + } +} + +async function handleRunCode(msg: RunCodeRequest) { + const { fileId, executionId } = msg + + const writtenPaths = new Set() + const readPaths = new Set() + + const computeImports = () => + [...readPaths].filter(path => !writtenPaths.has(path)) + + const postFailure = ( + stream: 'stderr' | 'info', + line: string, + errorType: ExecutionErrorType = 'generic' + ) => { + self.postMessage({ + type: 'output-line', + stream, + line, + fileId, + executionId, + }) + self.postMessage({ + type: 'run-code-result', + fileId, + executionId, + success: false, + outputs: [], + outputFiles: [], + imports: computeImports(), + errorType, + }) + } + + if (!pyodideModule || !pyodideIndexUrl) { + postFailure('stderr', 'Pyodide is not initialized') + return + } + + const instance = await pyodideModule.loadPyodide({ + env: { MPLBACKEND: 'Agg' }, + packageBaseUrl: `${pyodideIndexUrl}${pyodideModule.version}/`, + }) + + instance.setStdout({ + batched: (line: string) => { + self.postMessage({ + type: 'output-line', + stream: 'stdout', + line, + fileId, + executionId, + }) + }, + }) + instance.setStderr({ + batched: (line: string) => { + self.postMessage({ + type: 'output-line', + stream: 'stderr', + line, + fileId, + executionId, + }) + }, + }) + + const fs = instance.FS + const originalWrite = fs.write as PyodideFS['write'] + const originalRead = fs.read as PyodideFS['read'] + let runError: unknown = null + try { + if (msg.files.length > 0) { + syncProjectFiles(fs, msg.files) + } + + fs.write = ((...args: Parameters) => { + const [stream] = args + // Only surface writes to the synced project workspace, not Pyodide internals. + if ( + typeof stream?.path === 'string' && + stream.path.startsWith(PROJECT_FS_PREFIX) + ) { + writtenPaths.add(stream.path) + } + + return originalWrite.call(fs, ...args) + }) as PyodideFS['write'] + + fs.read = ((...args: Parameters) => { + const [stream] = args + if ( + typeof stream?.path === 'string' && + stream.path.startsWith(PROJECT_FS_PREFIX) + ) { + readPaths.add(stream.path) + } + + return originalRead.call(fs, ...args) + }) as PyodideFS['read'] + + await instance.loadPackagesFromImports(msg.code) + const result = await instance.runPythonAsync(msg.code) + if (result !== undefined) { + self.postMessage({ + type: 'output-line', + stream: 'stdout', + line: String(result), + fileId, + executionId, + }) + } + } catch (e) { + runError = e + } + fs.write = originalWrite + fs.read = originalRead + + const paths = [...writtenPaths] + + if (runError) { + const errorMessage = + runError instanceof Error ? runError.message : String(runError) + const errorType = classifyErrorType(errorMessage) + const fullMessage = + errorType === 'ModuleNotFoundError' + ? `${errorMessage}\n${moduleNotFoundHelpMessage()}` + : errorMessage + postFailure('stderr', fullMessage, errorType) + return + } + + const countViolation = checkOutputCount(paths.length) + if (countViolation) { + postFailure('info', countViolation.message, 'OutputLimitExceeded') + return + } + + const filesWithSizes: { path: string; size: number }[] = [] + for (const writtenPath of paths) { + try { + filesWithSizes.push({ + path: writtenPath, + size: fs.stat(writtenPath).size, + }) + } catch { + // A script can write a file and later delete or rename it before the run + // finishes; fs.stat would then throw and we'd never post a + // run-code-result, leaving the UI stuck. Skip paths we can't stat. + } + } + + const sizeViolation = checkOutputLimits(filesWithSizes) + if (sizeViolation) { + postFailure('info', sizeViolation.message, 'OutputLimitExceeded') + return + } + + const outputFiles: OutputFileData[] = [] + const transferables: Transferable[] = [] + for (const { path: writtenPath } of filesWithSizes) { + const content = fs.readFile(writtenPath) + const relativePath = writtenPath.slice(PROJECT_FS_PREFIX.length) + outputFiles.push({ relativePath, content }) + if (content.buffer instanceof ArrayBuffer) { + transferables.push(content.buffer) + } + } + + // The transferables moves ownership of each ArrayBuffer to the main thread + // instead of structured-cloning it. The buffers are already referenced from + // outputFiles.content; listing them here just swaps copy for move, so file + // contents travel through once rather than being allocated on both sides. + self.postMessage( + { + type: 'run-code-result', + fileId, + executionId, + success: true, + outputs: filesWithSizes.map(f => f.path), + outputFiles, + imports: computeImports(), + }, + transferables + ) +} + +self.addEventListener('message', async event => { + const msg = event.data as PyodideWorkerRequest + switch (msg.type) { + case 'init': + await handleInit(msg) + break + case 'run-code': + await handleRunCode(msg) + break + } +}) + +self.postMessage({ type: 'listening' }) diff --git a/services/web/modules/python-runner/frontend/js/components/editor/python/python-output-pane.tsx b/services/web/modules/python-runner/frontend/js/components/editor/python/python-output-pane.tsx new file mode 100644 index 0000000000..487fa56dea --- /dev/null +++ b/services/web/modules/python-runner/frontend/js/components/editor/python/python-output-pane.tsx @@ -0,0 +1,97 @@ +import { useEffect, useMemo, useSyncExternalStore } from 'react' +import { useTranslation } from 'react-i18next' +import classNames from 'classnames' +import OLButton from '@/shared/components/ol/ol-button' +import OLButtonToolbar from '@/shared/components/ol/ol-button-toolbar' +import MaterialIcon from '@/shared/components/material-icon' +import SplitTestBadge from '@/shared/components/split-test-badge' +import { sendMB } from '@/infrastructure/event-tracking' +import { useEditorOpenDocContext } from '@/features/ide-react/context/editor-open-doc-context' +import { usePythonExecutionContext } from '../../../context/python-execution-context' +import { DEFAULT_STATE } from './python-runner' + +const emptySubscribe = () => () => {} +const getDefaultState = () => DEFAULT_STATE + +export default function PythonOutputPane() { + const { t } = useTranslation() + const { currentDocumentId, openDocName } = useEditorOpenDocContext() + const { getPythonRunner } = usePythonExecutionContext() + const pythonRunner = useMemo( + () => (currentDocumentId ? getPythonRunner(currentDocumentId) : null), + [currentDocumentId, getPythonRunner] + ) + + useEffect(() => { + if (!currentDocumentId || !openDocName) { + return + } + sendMB('script-runner-opened', { + fileName: openDocName, + fileExtension: 'py', + editorMode: 'code', + }) + }, [currentDocumentId, openDocName]) + + const { output, error, status } = useSyncExternalStore( + pythonRunner ? pythonRunner.subscribe : emptySubscribe, + pythonRunner ? pythonRunner.getState : getDefaultState + ) + + if (!pythonRunner) { + return null + } + + return ( +
+ +
+
+ status === 'running' ? pythonRunner.interrupt() : pythonRunner.run()} + variant={status === 'running' ? 'danger' : 'primary'} + disabled={status === 'loading'} + aria-label={ + status === 'running' + ? t('stop_python_execution') + : t('run_python_code') + } + trailingIcon={status === 'running' ? 'stop' : 'play_arrow'} + > + {status === 'running' ? t('stop') : t('run')} + +
+ +
+ + +
+ {status === 'loading' && ( +
+ {t('loading_python_runtime')} +
+ )} + {status !== 'loading' && !error && output.length === 0 && ( +
+ {t('run_current_script_to_see_output')} +
+ )} + {error && ( +
{error}
+ )} + {output.map((entry, index) => ( +
+ {entry.line} +
+ ))} +
+
+ ) +} diff --git a/services/web/modules/python-runner/frontend/js/components/editor/python/python-output-toasts.tsx b/services/web/modules/python-runner/frontend/js/components/editor/python/python-output-toasts.tsx new file mode 100644 index 0000000000..c8b766041f --- /dev/null +++ b/services/web/modules/python-runner/frontend/js/components/editor/python/python-output-toasts.tsx @@ -0,0 +1,48 @@ +import { GlobalToastGeneratorEntry } from '@/features/ide-react/components/global-toasts' +import { useTranslation } from 'react-i18next' + +const stripProjectPrefix = (path: string) => path.replace(/^\/project\/?/, '') + +const PythonFilesSavedToast = ({ paths }: { paths: string[] }) => { + const { t } = useTranslation() + if (paths.length === 1) { + return ( + + {t('x_saved_to_your_project', { + fileName: stripProjectPrefix(paths[0]), + })} + + ) + } + return ( + {t('x_files_saved_to_your_project', { count: paths.length })} + ) +} + +const isStringArray = (value: unknown): value is string[] => + Array.isArray(value) && value.every(v => typeof v === 'string') + +const generators: GlobalToastGeneratorEntry[] = [ + { + key: 'python:files-saved', + generator: ({ paths }) => ({ + content: ( + + ), + type: 'success', + autoHide: true, + delay: 5000, + isDismissible: true, + }), + }, +] + +export default generators + +export const showPythonFilesSavedToast = (paths: string[]) => { + window.dispatchEvent( + new CustomEvent('ide:show-toast', { + detail: { key: 'python:files-saved', paths }, + }) + ) +} diff --git a/services/web/modules/python-runner/frontend/js/components/editor/python/python-runner.ts b/services/web/modules/python-runner/frontend/js/components/editor/python/python-runner.ts new file mode 100644 index 0000000000..81fe7c042b --- /dev/null +++ b/services/web/modules/python-runner/frontend/js/components/editor/python/python-runner.ts @@ -0,0 +1,305 @@ +// Per-file Python execution manager. Each PythonRunner owns a PyodideWorkerClient +// and exposes a subscribe/getState API for use with useSyncExternalStore, +// so React components can reactively read execution status and output. +import path from 'path-browserify' +import { v4 as uuid } from 'uuid' +import { debugConsole } from '@/utils/debugging' +import { sendMB } from '@/infrastructure/event-tracking' +import { PyodideWorkerClient } from './pyodide-worker-client' +import { showPythonFilesSavedToast } from './python-output-toasts' +import type { OutputStream } from './pyodide-worker-messages' +import type { + BatchUploadItem, + UploadResult, +} from '@/infrastructure/batch-file-uploader' + +export type FileUploader = (items: BatchUploadItem[]) => Promise + +const MAX_OUTPUT_LINES = 100 +const PROJECT_FS_PREFIX = '/project/' + +function stripProjectFsPrefix(p: string): string { + return p.startsWith(PROJECT_FS_PREFIX) ? p.slice(PROJECT_FS_PREFIX.length) : p +} + +export type ExecutionStatus = + | 'loading' + | 'idle' + | 'running' + | 'finished' + | 'errored' + +export type ExecutionContext = { + code: string + files: { relativePath: string; content: string }[] +} + +type Listener = () => void + +export type OutputLine = { + stream: OutputStream + line: string +} + +export type PythonRunnerState = { + output: OutputLine[] + status: ExecutionStatus + error: string | null +} + +export const DEFAULT_STATE: PythonRunnerState = { + output: [], + status: 'loading', + error: null, +} + +export class PythonRunner { + readonly fileId: string + private client: PyodideWorkerClient | null = null + private readonly baseAssetPath: string + private readonly createWorker: () => Worker + private readonly getExecutionContext: () => Promise + private readonly fileUploader: FileUploader + + private listeners = new Set() + + private activeExecution: { id: string; startedAt: number } | null = null + private state: PythonRunnerState = DEFAULT_STATE + + constructor( + fileId: string, + baseAssetPath: string, + getExecutionContext: () => Promise, + createWorker: () => Worker, + fileUploader: FileUploader + ) { + this.fileId = fileId + this.baseAssetPath = baseAssetPath + this.createWorker = createWorker + this.getExecutionContext = getExecutionContext + this.fileUploader = fileUploader + } + + subscribe = (listener: Listener): (() => void) => { + this.listeners.add(listener) + return () => this.listeners.delete(listener) + } + + getState = () => this.state + + private updateState(fields: Partial) { + const prev = this.state + const output = fields.output ?? prev.output + const status = fields.status ?? prev.status + const error = fields.error !== undefined ? fields.error : prev.error + + if ( + output === prev.output && + status === prev.status && + error === prev.error + ) { + return + } + + this.state = { output, status, error } + + for (const listener of this.listeners) { + listener() + } + } + + init() { + if (this.client) { + return + } + + this.updateState({ status: 'loading', error: null }) + + this.client = new PyodideWorkerClient({ + baseAssetPath: this.baseAssetPath, + createWorker: this.createWorker, + fileUploader: this.fileUploader, + onLifecycle: event => { + switch (event.type) { + case 'loaded': + this.updateState({ status: 'idle', error: null }) + return + + case 'loading-failed': + debugConsole.error('Failed to load Python runtime', event.error) + this.updateState({ status: 'errored', error: event.error }) + return + + case 'run-finished': { + const active = this.activeExecution + if ( + event.fileId !== this.fileId || + active?.id !== event.executionId + ) { + return + } + + this.activeExecution = null + + sendMB('script-runner-execution-completed', { + result: event.success ? 'success' : 'error', + errorType: event.success ? undefined : event.errorType, + executionTimeMs: Math.round(performance.now() - active.startedAt), + filesImportedCount: event.imports.length, + filesImportedExtensions: collectExtensions(event.imports), + filesWrittenCount: event.outputs.length, + filesWrittenExtensions: collectExtensions(event.outputs), + }) + + // event.outputs are full worker paths (/project/foo.txt) while + // event.failedUploads are relativePaths (foo.txt); strip the + // prefix before comparing. + const failed = new Set(event.failedUploads) + const uploadedPaths = event.outputs + .map(stripProjectFsPrefix) + .filter(p => !failed.has(p)) + if (uploadedPaths.length > 0) { + showPythonFilesSavedToast(uploadedPaths) + } + + this.updateState({ status: 'finished' }) + } + } + }, + onOutput: (stream, line, fileId, executionId) => { + if ( + fileId !== this.fileId || + this.activeExecution?.id !== executionId + ) { + return + } + this.updateState({ + output: appendCapped(this.state.output, { stream, line }), + }) + }, + }) + } + + async run() { + if (!this.client || this.state.status === 'loading') { + return + } + + let context: ExecutionContext | null + try { + context = await this.getExecutionContext() + } catch (err) { + debugConsole.error('Failed to build execution context', err) + this.updateState({ status: 'errored', error: formatError(err) }) + return + } + + // Re-check after await — status may have changed but TypeScript + // still narrows from the pre-await check, so we cast back. + if ( + !context || + !this.client || + (this.state.status as ExecutionStatus) === 'loading' + ) { + return + } + + const { code, files } = context + + sendMB('script-runner-run-clicked', { + scriptLineCount: countLines(code), + }) + + const executionId = uuid() + this.activeExecution = { id: executionId, startedAt: performance.now() } + this.updateState({ status: 'running', output: [], error: null }) + + try { + this.client.runCode(code, { + fileId: this.fileId, + executionId, + files, + }) + } catch (runError) { + if (this.activeExecution?.id !== executionId) { + return + } + this.activeExecution = null + this.updateState({ status: 'errored', error: formatError(runError) }) + } + } + + interrupt() { + if (!this.client) { + return + } + + if (this.state.status === 'running' && this.activeExecution) { + sendMB('script-runner-stop-clicked', { + timeBeforeStopMs: Math.round( + performance.now() - this.activeExecution.startedAt + ), + }) + } + + this.client.reset() + this.activeExecution = null + + // The worker is terminated and recreated by reset(), so it needs to + // reload Pyodide. The 'loaded' lifecycle callback will transition + // back to 'idle'. + this.updateState({ + status: 'loading', + output: + this.state.status === 'running' + ? appendCapped(this.state.output, { + stream: 'info', + line: 'Execution interrupted', + }) + : this.state.output, + }) + } + + destroy() { + if (this.client) { + this.client.destroy() + this.client = null + } + } +} + +function appendCapped(existing: OutputLine[], entry: OutputLine): OutputLine[] { + const updated = [...existing, entry] + return updated.length > MAX_OUTPUT_LINES + ? updated.slice(-MAX_OUTPUT_LINES) + : updated +} + +function formatError(error: unknown): string { + if (error instanceof Error) { + return error.message + } + return String(error) +} + +function countLines(code: string): number { + if (code.length === 0) { + return 0 + } + return code.split('\n').length +} + +function extractExtension(filePath: string): string { + return path.extname(filePath).slice(1).toLowerCase() +} + +function collectExtensions(filePaths: string[]): string { + const seen = new Set() + for (const filePath of filePaths) { + const ext = extractExtension(filePath) + if (ext) { + seen.add(ext) + } + } + return Array.from(seen).join(',') +} diff --git a/services/web/modules/python-runner/frontend/js/components/layout/python-editor-split.tsx b/services/web/modules/python-runner/frontend/js/components/layout/python-editor-split.tsx new file mode 100644 index 0000000000..89ec3f6d66 --- /dev/null +++ b/services/web/modules/python-runner/frontend/js/components/layout/python-editor-split.tsx @@ -0,0 +1,31 @@ +import '../../../stylesheets/python-runner.scss' +import { Panel, PanelGroup } from 'react-resizable-panels' +import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle' +import PythonOutputPane from '../editor/python/python-output-pane' +import SourceEditor from '@/features/source-editor/components/source-editor' +import { PythonExecutionProvider } from '../../context/python-execution-context' + +export const PythonEditorSplit = () => { + return ( + + + + + + + + + + + + ) +} diff --git a/services/web/modules/python-runner/frontend/js/context/python-execution-context.tsx b/services/web/modules/python-runner/frontend/js/context/python-execution-context.tsx new file mode 100644 index 0000000000..5575179e76 --- /dev/null +++ b/services/web/modules/python-runner/frontend/js/context/python-execution-context.tsx @@ -0,0 +1,167 @@ +import { + createContext, + FC, + PropsWithChildren, + useCallback, + useContext, + useEffect, + useMemo, + useRef, +} from 'react' +import getMeta from '@/utils/meta' +import { useFileTreePathContext } from '@/features/file-tree/contexts/file-tree-path' +import { useEditorManagerContext } from '@/features/ide-react/context/editor-manager-context' +import { useProjectContext } from '@/shared/context/project-context' +import { useFileTreeData } from '@/shared/context/file-tree-data-context' +import { + uploadBatch, + BatchUploadItem, +} from '@/infrastructure/batch-file-uploader' +import { + PythonRunner, + ExecutionContext, +} from '../components/editor/python/python-runner' + +// Worker factory lives here (a .tsx file) so that the full +// `new Worker(new URL(..., import.meta.url))` expression is in a single place +// where webpack 5 can statically detect it and create a proper worker bundle. +// Keeping import.meta.url out of .ts files also avoids Node.js 24 switching to +// ESM mode and breaking CJS-based test loading via @babel/register. +const createPyodideWorker = () => + new Worker( + /* webpackChunkName: "pyodide-worker" */ + new URL('../components/editor/python/pyodide.worker.ts', import.meta.url), + { type: 'module' } + ) + +export interface PythonExecutionContextValue { + getPythonRunner: (fileId: string) => PythonRunner +} + +export const PythonExecutionContext = createContext< + PythonExecutionContextValue | undefined +>(undefined) + +export const PythonExecutionProvider: FC = ({ + children, +}) => { + const { openDocs } = useEditorManagerContext() + const { projectId, projectSnapshot } = useProjectContext() + const { pathInFolder } = useFileTreePathContext() + const { fileTreeData } = useFileTreeData() + const runnersRef = useRef(new Map()) + const baseAssetPathRef = useRef(null) + + const pathInFolderRef = useRef(pathInFolder) + pathInFolderRef.current = pathInFolder + + // Ref so the upload closure built into each PythonRunner reads the + // current value at call time rather than capturing a potentially-stale + // value from when the runner was constructed (fileTreeData may load + // after the runner is created). + const fileTreeDataRef = useRef(fileTreeData) + fileTreeDataRef.current = fileTreeData + + // Refreshes the project snapshot and resolves the source code and all project + // files for the given fileId, to be passed to the executor for running. + const getExecutionContext = useCallback( + async (fileId: string): Promise => { + await openDocs.awaitBufferedOps(AbortSignal.timeout(5000)) + await projectSnapshot.refresh() + + const relativePath = pathInFolderRef.current(fileId) + if (!relativePath) { + return null + } + + const code = projectSnapshot.getDocContents(relativePath) + if (code == null) { + return null + } + + const docPaths = projectSnapshot.getDocPaths() + const files = docPaths + .map(docPath => { + const content = projectSnapshot.getDocContents(docPath) + return content != null ? { relativePath: docPath, content } : null + }) + .filter( + (f): f is { relativePath: string; content: string } => f != null + ) + + return { code, files } + }, + [openDocs, projectSnapshot] + ) + + const getPythonRunner = useCallback( + (fileId: string): PythonRunner => { + const existing = runnersRef.current.get(fileId) + if (existing) { + return existing + } + + if (!baseAssetPathRef.current) { + baseAssetPathRef.current = new URL( + getMeta('ol-baseAssetPath'), + window.location.href + ).toString() + } + + const uploadOutputFiles = (items: BatchUploadItem[]) => { + const folderId = fileTreeDataRef.current?._id + if (!folderId) { + return Promise.reject( + new Error('File tree not loaded; cannot upload output files') + ) + } + return uploadBatch(items, { + projectId, + folderId, + }) + } + + const runner = new PythonRunner( + fileId, + baseAssetPathRef.current, + () => getExecutionContext(fileId), + createPyodideWorker, + uploadOutputFiles + ) + runner.init() + runnersRef.current.set(fileId, runner) + return runner + }, + [getExecutionContext, projectId] + ) + + useEffect(() => { + const runners = runnersRef.current + return () => { + for (const runner of runners.values()) { + runner.destroy() + } + runners.clear() + } + }, []) + + const value = useMemo(() => ({ getPythonRunner }), [getPythonRunner]) + + return ( + + {children} + + ) +} + +export const usePythonExecutionContext = (): PythonExecutionContextValue => { + const context = useContext(PythonExecutionContext) + + if (!context) { + throw new Error( + 'usePythonExecutionContext is only available inside PythonExecutionContext.Provider' + ) + } + + return context +} diff --git a/services/web/modules/python-runner/frontend/stylesheets/python-runner.scss b/services/web/modules/python-runner/frontend/stylesheets/python-runner.scss new file mode 100644 index 0000000000..f57d318fb6 --- /dev/null +++ b/services/web/modules/python-runner/frontend/stylesheets/python-runner.scss @@ -0,0 +1,71 @@ +.ide-redesign-python-editor-split { + height: 100%; +} + +.ide-redesign-python-output-pane { + height: 100%; + display: flex; + flex-direction: column; + background-color: var(--ide-redesign-background); + color: var(--ide-redesign-color); + border-top: 1px solid var(--border-divider); +} + +.ide-redesign-python-output-pane-toolbar { + display: flex; + align-items: center; + height: var(--toolbar-height); + border-bottom: 1px solid var(--toolbar-border-color); +} + +.ide-redesign-python-output-pane-toolbar-left { + display: flex; + align-items: center; + align-self: stretch; + gap: var(--spacing-02); +} + +.ide-redesign-python-output-pane-run-button-wrapper { + margin-left: var(--spacing-04); + + .btn-primary:hover { + z-index: auto; + } +} + +.ide-redesign-python-output-pane-body { + flex: 1; + overflow: auto; + padding: var(--spacing-06); + font-family: DM Mono, monospace; + font-size: var(--font-size-02); + line-height: var(--line-height-03); +} + +.ide-redesign-python-output-pane-placeholder { + color: var(--content-secondary); +} + +.ide-redesign-python-output-pane-line { + white-space: pre-wrap; + word-break: break-word; + min-height: var(--line-height-03); +} + +.ide-redesign-python-output-pane-line-info { + color: var(--content-info); +} + +.ide-redesign-python-output-pane-line-stderr { + color: var(--red-50); +} + +.ide-redesign-python-output-pane-placeholder, +.ide-redesign-python-output-pane-error { + white-space: pre-wrap; +} + +.ide-redesign-python-output-pane-error { + margin-bottom: var(--spacing-03); + color: var(--red-50); +} diff --git a/services/web/modules/python-runner/index.mjs b/services/web/modules/python-runner/index.mjs new file mode 100644 index 0000000000..8b2b78554d --- /dev/null +++ b/services/web/modules/python-runner/index.mjs @@ -0,0 +1,6 @@ +import logger from '@overleaf/logger' + +logger.debug({}, 'Enable Python Script Runner') +const pythonRunnerModule = {} + +export default pythonRunnerModule diff --git a/services/web/modules/python-runner/test/frontend/components/python-output-pane.spec.tsx b/services/web/modules/python-runner/test/frontend/components/python-output-pane.spec.tsx new file mode 100644 index 0000000000..2eb278a5cd --- /dev/null +++ b/services/web/modules/python-runner/test/frontend/components/python-output-pane.spec.tsx @@ -0,0 +1,481 @@ +import React, { FC, PropsWithChildren } from 'react' +import PythonOutputPane from '../../../frontend/js/components/editor/python/python-output-pane' +import { + EditorProviders, + projectDefaults, +} from '../../../helpers/editor-providers' +import { FileTreePathContext } from '@/features/file-tree/contexts/file-tree-path' +import { ProjectContext } from '@/shared/context/project-context' +import { ProjectSnapshot } from '@/infrastructure/project-snapshot' +import { PythonExecutionProvider } from '../../../frontend/js/context/python-execution-context' + +const pythonExecutableScript: Record = { + file_id: 'test-py-doc-id', + filename: 'test.py', +} + +const FileTreePathProvider: FC = ({ children }) => { + return ( + pythonExecutableScript.filename, + findEntityByPath: () => null, + previewByPath: () => null, + dirname: () => null, + }} + > + {children} + + ) +} + +function makeProjectProvider(fileContents: Record) { + const ProjectProvider: FC = ({ children }) => { + const projectSnapshot = { + refresh: async () => {}, + getDocPaths: () => Object.keys(fileContents), + getDocContents: (path: string) => fileContents[path] ?? null, + } as unknown as ProjectSnapshot + + return ( + {}, + updateProject: () => {}, + joinedOnce: true, + projectSnapshot, + tags: [], + features: projectDefaults.features, + name: projectDefaults.name, + }} + > + {children} + + ) + } + return ProjectProvider +} + +describe('', function () { + beforeEach(function () { + window.metaAttributesCache.set('ol-baseAssetPath', '/__cypress/src/') + }) + + it('executes a Python script and displays its output', function () { + const executablePythonFileContents = "print('hello!')" + const projectFiles = { + [pythonExecutableScript.filename]: executablePythonFileContents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: pythonExecutableScript.filename, + }, + }} + providers={{ FileTreePathProvider, ProjectProvider }} + > + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + cy.findByText('hello!').should('exist') + }) + + it('can import and use values from other project Python files', function () { + const executablePythonFileContents = + 'from message import message\nprint(message)' + + const importedPythonFile = { + filename: 'message.py', + file_contents: "message = 'hello!'", + } + + const projectFiles = { + [pythonExecutableScript.filename]: executablePythonFileContents, + [importedPythonFile.filename]: importedPythonFile.file_contents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: pythonExecutableScript.filename, + }, + }} + providers={{ FileTreePathProvider, ProjectProvider }} + > + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + cy.findByText('hello!').should('exist') + }) + + it('can import files from different directories relative to the executable script', function () { + const executablePythonFileContents = [ + 'from scripts.data_importers.csv_importer import print_data', + 'print_data()', + ].join('\n') + + const csvImporterFile = { + filename: 'scripts/data_importers/csv_importer.py', + file_contents: [ + 'import csv', + '', + 'def print_data():', + ' with open("food_items.csv", "r") as f:', + ' reader = csv.reader(f)', + ' for row in reader:', + ' print(",".join(row))', + ].join('\n'), + } + + const csvDataFile = { + filename: 'food_items.csv', + file_contents: 'name,type\nPizza,Italian\nSushi,Japanese\nTacos,Mexican', + } + + const projectFiles = { + 'scripts/test.py': executablePythonFileContents, + [csvImporterFile.filename]: csvImporterFile.file_contents, + [csvDataFile.filename]: csvDataFile.file_contents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + const NestedFileTreePathProvider: FC = ({ + children, + }) => ( + 'scripts/test.py', + findEntityByPath: () => null, + previewByPath: () => null, + dirname: () => null, + }} + > + {children} + + ) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: 'test.py', + }, + }} + providers={{ + FileTreePathProvider: NestedFileTreePathProvider, + ProjectProvider, + }} + > + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + cy.findByText('name,type').should('exist') + cy.findByText('Pizza,Italian').should('exist') + cy.findByText('Sushi,Japanese').should('exist') + cy.findByText('Tacos,Mexican').should('exist') + }) + + it('renders stderr output with the stderr line class', function () { + const executablePythonFileContents = [ + 'import sys', + "print('hello!')", + "sys.stderr.write('boom\\n')", + ].join('\n') + const projectFiles = { + [pythonExecutableScript.filename]: executablePythonFileContents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: pythonExecutableScript.filename, + }, + }} + providers={{ FileTreePathProvider, ProjectProvider }} + > + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + + cy.findByText('hello!') + .should('have.class', 'ide-redesign-python-output-pane-line-stdout') + .and('not.have.class', 'ide-redesign-python-output-pane-line-stderr') + cy.findByText('boom').should( + 'have.class', + 'ide-redesign-python-output-pane-line-stderr' + ) + cy.findByText("Only Pyodide's built-in packages", { exact: false }).should( + 'not.exist' + ) + }) + + it('renders the interrupt message as an info line', function () { + const executablePythonFileContents = 'while True:\n pass\n' + const projectFiles = { + [pythonExecutableScript.filename]: executablePythonFileContents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: pythonExecutableScript.filename, + }, + }} + providers={{ FileTreePathProvider, ProjectProvider }} + > + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + cy.findByRole('button', { name: 'Stop Python execution' }) + .should('not.be.disabled') + .click() + + cy.findByText('Execution interrupted').should( + 'have.class', + 'ide-redesign-python-output-pane-line-info' + ) + }) + + it('renders stdout, stderr, and info lines with visually distinct CSS classes', function () { + const executablePythonFileContents = [ + 'import sys', + "print('hello!')", + "sys.stderr.write('boom\\n')", + 'while True:', + ' pass', + ].join('\n') + const projectFiles = { + [pythonExecutableScript.filename]: executablePythonFileContents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: pythonExecutableScript.filename, + }, + }} + providers={{ FileTreePathProvider, ProjectProvider }} + > + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + + cy.findByText('hello!') + .should('have.class', 'ide-redesign-python-output-pane-line-stdout') + .and('not.have.class', 'ide-redesign-python-output-pane-line-stderr') + .and('not.have.class', 'ide-redesign-python-output-pane-line-info') + cy.findByText('boom') + .should('have.class', 'ide-redesign-python-output-pane-line-stderr') + .and('not.have.class', 'ide-redesign-python-output-pane-line-stdout') + .and('not.have.class', 'ide-redesign-python-output-pane-line-info') + + cy.findByRole('button', { name: 'Stop Python execution' }) + .should('not.be.disabled') + .click() + + cy.findByText('Execution interrupted') + .should('have.class', 'ide-redesign-python-output-pane-line-info') + .and('not.have.class', 'ide-redesign-python-output-pane-line-stdout') + .and('not.have.class', 'ide-redesign-python-output-pane-line-stderr') + }) + + it('can load common python data analysis packages on code execution', function () { + const executablePythonFileContents = [ + 'import tomli', + '', + "print(tomli.loads('greeting = \"hello from tomli\"')['greeting'])", + ].join('\n') + + const projectFiles = { + [pythonExecutableScript.filename]: executablePythonFileContents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: pythonExecutableScript.filename, + }, + }} + providers={{ FileTreePathProvider, ProjectProvider }} + > + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + cy.findByText("ModuleNotFoundError: No module named 'tomli'").should( + 'not.exist' + ) + cy.findByText('hello from tomli').should('exist') + }) + + it('auto-installs python packages imported by the executing script', function () { + const executablePythonFileContents = [ + 'import tomli', + '', + "print(tomli.loads('greeting = \"hello from tomli\"')['greeting'])", + ].join('\n') + + const projectFiles = { + [pythonExecutableScript.filename]: executablePythonFileContents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: pythonExecutableScript.filename, + }, + }} + providers={{ FileTreePathProvider, ProjectProvider }} + > + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + cy.findByText("ModuleNotFoundError: No module named 'tomli'").should( + 'not.exist' + ) + cy.findByText('hello from tomli').should('exist') + }) + + it('augments ModuleNotFoundError output with help text about supported Pyodide packages', function () { + const executablePythonFileContents = + 'import this_module_does_not_exist_in_pyodide\n' + const projectFiles = { + [pythonExecutableScript.filename]: executablePythonFileContents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: pythonExecutableScript.filename, + }, + }} + providers={{ FileTreePathProvider, ProjectProvider }} + > + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + + cy.findByText('ModuleNotFoundError', { exact: false }).should('exist') + cy.findByText("Only Pyodide's built-in packages", { exact: false }).should( + 'have.class', + 'ide-redesign-python-output-pane-line-stderr' + ) + }) +}) diff --git a/services/web/modules/python-runner/test/unit/editor/pyodide-worker-client.spec.ts b/services/web/modules/python-runner/test/unit/editor/pyodide-worker-client.spec.ts new file mode 100644 index 0000000000..288041db8e --- /dev/null +++ b/services/web/modules/python-runner/test/unit/editor/pyodide-worker-client.spec.ts @@ -0,0 +1,778 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import { + PyodideWorkerClient, + type LifecycleCallback, + type OutputCallback, +} from '../../../frontend/js/components/editor/python/pyodide-worker-client' +import type { FileUploader } from '../../../frontend/js/components/editor/python/python-runner' +import { WorkerMock, createWorker } from './worker-mock' + +const BASE_ASSET_PATH = 'https://assets.example.test/' + +const fileUploaderStub: FileUploader = () => Promise.resolve([]) + +describe('PyodideWorkerClient', function () { + beforeEach(function () { + WorkerMock.instances.length = 0 + }) + + it('queues runCode until the worker reports listening', function () { + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + fileUploader: fileUploaderStub, + }) + const worker = WorkerMock.instances[0] + + client.runCode('print("ok")', { + fileId: 'main.py', + executionId: 'exec-1', + files: [{ relativePath: 'main.py', content: 'print("ok")' }], + }) + expect(worker.postedMessages).to.have.length(0) + + worker.emitMessage({ type: 'listening' }) + expect(worker.postedMessages.map(message => message.type)).to.deep.equal([ + 'init', + 'run-code', + ]) + + const runRequest = worker.postedMessages.find( + message => message.type === 'run-code' + ) + expect(runRequest).to.include({ + type: 'run-code', + fileId: 'main.py', + executionId: 'exec-1', + code: 'print("ok")', + }) + expect(runRequest.files).to.deep.equal([ + { relativePath: 'main.py', content: 'print("ok")' }, + ]) + }) + + it('sends runCode as fire-and-forget', function () { + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + fileUploader: fileUploaderStub, + }) + const worker = WorkerMock.instances[0] + worker.emitMessage({ type: 'listening' }) + + client.runCode('raise RuntimeError("boom")', { + fileId: 'boom.py', + executionId: 'exec-2', + files: [], + }) + const runRequest = worker.postedMessages.find( + message => message.type === 'run-code' + ) + expect(runRequest).to.include({ + type: 'run-code', + fileId: 'boom.py', + executionId: 'exec-2', + }) + }) + + function setupClientWithLifecycleTracking() { + const lifecycleEvents: Parameters[0][] = [] + + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + onLifecycle: event => { + lifecycleEvents.push(event) + }, + fileUploader: fileUploaderStub, + }) + const worker = WorkerMock.instances[0] + worker.emitMessage({ type: 'listening' }) + return { client, worker, lifecycleEvents } + } + + function setupClientWithUploadTracking(options: { + fileUploader: FileUploader + }) { + const lifecycleEvents: Parameters[0][] = [] + const outputCalls: Parameters[] = [] + + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + onLifecycle: event => { + lifecycleEvents.push(event) + }, + onOutput: (...args) => { + outputCalls.push(args) + }, + fileUploader: options.fileUploader, + }) + const worker = WorkerMock.instances[0] + worker.emitMessage({ type: 'listening' }) + return { client, worker, lifecycleEvents, outputCalls } + } + + async function waitFor(predicate: () => boolean, timeoutMs = 200) { + const deadline = Date.now() + timeoutMs + while (!predicate()) { + if (Date.now() > deadline) { + throw new Error('waitFor timed out') + } + await new Promise(resolve => setTimeout(resolve, 0)) + } + } + + it('emits run-finished lifecycle event from run-code-result', function () { + const { client, worker, lifecycleEvents } = + setupClientWithLifecycleTracking() + + client.runCode('print("ok")', { + fileId: 'main.py', + executionId: 'exec-3', + files: [], + }) + worker.emitMessage({ + type: 'run-code-result', + fileId: 'main.py', + executionId: 'exec-3', + success: true, + outputs: ['/project/output.txt'], + outputFiles: [], + imports: [], + }) + + expect(lifecycleEvents).to.deep.equal([ + { + type: 'run-finished', + fileId: 'main.py', + executionId: 'exec-3', + success: true, + outputs: ['/project/output.txt'], + failedUploads: [], + imports: [], + errorType: undefined, + }, + ]) + }) + + it('surfaces outputs array from run-code-result with multiple files', function () { + const { client, worker, lifecycleEvents } = + setupClientWithLifecycleTracking() + + client.runCode('write_files()', { + fileId: 'main.py', + executionId: 'exec-4', + files: [], + }) + worker.emitMessage({ + type: 'run-code-result', + fileId: 'main.py', + executionId: 'exec-4', + success: true, + outputs: ['/project/fig1.png', '/project/results/data.csv'], + outputFiles: [], + imports: [], + }) + + expect(lifecycleEvents).to.deep.equal([ + { + type: 'run-finished', + fileId: 'main.py', + executionId: 'exec-4', + success: true, + outputs: ['/project/fig1.png', '/project/results/data.csv'], + failedUploads: [], + imports: [], + errorType: undefined, + }, + ]) + }) + + it('surfaces empty outputs when no project files were written', function () { + const { client, worker, lifecycleEvents } = + setupClientWithLifecycleTracking() + + client.runCode('print("no writes")', { + fileId: 'main.py', + executionId: 'exec-5', + files: [], + }) + worker.emitMessage({ + type: 'run-code-result', + fileId: 'main.py', + executionId: 'exec-5', + success: true, + outputs: [], + outputFiles: [], + imports: [], + }) + + expect(lifecycleEvents).to.deep.equal([ + { + type: 'run-finished', + fileId: 'main.py', + executionId: 'exec-5', + success: true, + outputs: [], + failedUploads: [], + imports: [], + errorType: undefined, + }, + ]) + }) + + it('surfaces success and outputFiles from run-code-result', async function () { + const { client, worker, lifecycleEvents } = + setupClientWithLifecycleTracking() + + client.runCode('write_files()', { + fileId: 'main.py', + executionId: 'exec-success', + files: [], + }) + const csvContent = new Uint8Array([1, 2, 3]) + const pngContent = new Uint8Array([4, 5, 6, 7]) + worker.emitMessage({ + type: 'run-code-result', + fileId: 'main.py', + executionId: 'exec-success', + success: true, + outputs: ['/project/data.csv', '/project/plot.png'], + outputFiles: [ + { relativePath: 'data.csv', content: csvContent }, + { relativePath: 'plot.png', content: pngContent }, + ], + imports: [], + }) + + await waitFor(() => + Boolean(lifecycleEvents.find(e => e.type === 'run-finished')) + ) + + const finished = lifecycleEvents.find(e => e.type === 'run-finished') + expect(finished).to.deep.equal({ + type: 'run-finished', + fileId: 'main.py', + executionId: 'exec-success', + success: true, + outputs: ['/project/data.csv', '/project/plot.png'], + failedUploads: [], + imports: [], + errorType: undefined, + }) + }) + + it('propagates ModuleNotFoundError errorType on run-finished', function () { + const { client, worker, lifecycleEvents } = + setupClientWithLifecycleTracking() + + client.runCode('import nope', { + fileId: 'main.py', + executionId: 'exec-mnfe', + files: [], + }) + worker.emitMessage({ + type: 'run-code-result', + fileId: 'main.py', + executionId: 'exec-mnfe', + success: false, + outputs: [], + outputFiles: [], + imports: [], + errorType: 'ModuleNotFoundError', + }) + + const finished = lifecycleEvents.find(e => e.type === 'run-finished') + expect(finished).to.deep.equal({ + type: 'run-finished', + fileId: 'main.py', + executionId: 'exec-mnfe', + success: false, + outputs: [], + failedUploads: [], + imports: [], + errorType: 'ModuleNotFoundError', + }) + }) + + it('surfaces success: false with empty outputFiles on script error', function () { + const { client, worker, lifecycleEvents } = + setupClientWithLifecycleTracking() + + client.runCode('raise RuntimeError("boom")', { + fileId: 'main.py', + executionId: 'exec-error', + files: [], + }) + worker.emitMessage({ + type: 'run-code-result', + fileId: 'main.py', + executionId: 'exec-error', + success: false, + outputs: [], + outputFiles: [], + imports: [], + errorType: 'generic', + }) + + const finished = lifecycleEvents.find(e => e.type === 'run-finished') + expect(finished).to.deep.equal({ + type: 'run-finished', + fileId: 'main.py', + executionId: 'exec-error', + success: false, + outputs: [], + failedUploads: [], + imports: [], + errorType: 'generic', + }) + }) + + it('surfaces empty outputFiles when success but no files were written', function () { + const { client, worker, lifecycleEvents } = + setupClientWithLifecycleTracking() + + client.runCode('print("no writes")', { + fileId: 'main.py', + executionId: 'exec-nowrites', + files: [], + }) + worker.emitMessage({ + type: 'run-code-result', + fileId: 'main.py', + executionId: 'exec-nowrites', + success: true, + outputs: [], + outputFiles: [], + imports: [], + }) + + const finished = lifecycleEvents.find(e => e.type === 'run-finished') + expect(finished).to.deep.equal({ + type: 'run-finished', + fileId: 'main.py', + executionId: 'exec-nowrites', + success: true, + outputs: [], + failedUploads: [], + imports: [], + errorType: undefined, + }) + }) + + it('reports lifecycle failure and rejects future run requests when loading fails', function () { + const lifecycleEvents: { type: string; error?: string }[] = [] + + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + onLifecycle: event => { + lifecycleEvents.push(event) + }, + fileUploader: fileUploaderStub, + }) + const worker = WorkerMock.instances[0] + + worker.emitMessage({ + type: 'loading-failed', + error: 'runtime unavailable', + }) + + expect(lifecycleEvents).to.deep.equal([ + { type: 'loading-failed', error: 'runtime unavailable' }, + ]) + expect(() => + client.runCode('print("ok")', { + fileId: 'main.py', + executionId: 'exec-4', + files: [], + }) + ).to.throw('runtime unavailable') + }) + + it('terminates the worker even when destroy is called after loading failure', function () { + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + fileUploader: fileUploaderStub, + }) + const worker = WorkerMock.instances[0] + + worker.emitMessage({ + type: 'loading-failed', + error: 'runtime unavailable', + }) + client.destroy() + + expect(worker.terminated).to.equal(true) + }) + + context('upload behavior', function () { + const successResult = (name: string, relativePath: string) => ({ + status: 'success' as const, + name, + relativePath, + data: { success: true }, + }) + const errorResult = ( + name: string, + relativePath: string, + error: string + ) => ({ + status: 'error' as const, + name, + relativePath, + error, + }) + + function emitRunResult( + worker: WorkerMock, + executionId: string, + outputFiles: Array<{ relativePath: string; content: Uint8Array }>, + success = true + ) { + worker.emitMessage({ + type: 'run-code-result', + fileId: 'main.py', + executionId, + success, + outputs: [], + outputFiles, + imports: [], + }) + } + + const findFinished = ( + lifecycleEvents: Parameters[0][] + ) => lifecycleEvents.find(e => e.type === 'run-finished') + + it('invokes fileUploader with mapped items when run-code-result has output files', async function () { + const uploader = sinon + .stub() + .resolves([successResult('data.csv', 'output/data.csv')]) + const { worker, lifecycleEvents } = setupClientWithUploadTracking({ + fileUploader: uploader, + }) + + emitRunResult(worker, 'exec-up-1', [ + { + relativePath: 'output/data.csv', + content: new TextEncoder().encode('a,b\n1,2'), + }, + ]) + await waitFor(() => Boolean(findFinished(lifecycleEvents))) + + expect(uploader.calledOnce).to.be.true + const [items] = uploader.firstCall.args + expect(items).to.have.lengthOf(1) + expect(items[0].name).to.equal('data.csv') + expect(items[0].relativePath).to.equal('output/data.csv') + expect(items[0].file).to.be.instanceOf(Blob) + }) + + it('emits success: true and empty failedUploads when all uploads succeed', async function () { + const uploader = sinon + .stub() + .resolves([ + successResult('a.csv', 'a.csv'), + successResult('b.csv', 'b.csv'), + ]) + const { worker, lifecycleEvents } = setupClientWithUploadTracking({ + fileUploader: uploader, + }) + + emitRunResult(worker, 'exec-up-2', [ + { relativePath: 'a.csv', content: new TextEncoder().encode('1') }, + { relativePath: 'b.csv', content: new TextEncoder().encode('2') }, + ]) + await waitFor(() => Boolean(findFinished(lifecycleEvents))) + + const finished = findFinished(lifecycleEvents) + expect(finished).to.deep.include({ success: true, failedUploads: [] }) + }) + + it('flips success to false and lists failed paths when an upload fails', async function () { + const uploader = sinon + .stub() + .resolves([ + successResult('good.csv', 'good.csv'), + errorResult('bad.csv', 'output/bad.csv', 'duplicate_file_name'), + ]) + const { worker, lifecycleEvents, outputCalls } = + setupClientWithUploadTracking({ fileUploader: uploader }) + + emitRunResult(worker, 'exec-up-3', [ + { relativePath: 'good.csv', content: new TextEncoder().encode('1') }, + { + relativePath: 'output/bad.csv', + content: new TextEncoder().encode('2'), + }, + ]) + await waitFor(() => Boolean(findFinished(lifecycleEvents))) + + const finished = findFinished(lifecycleEvents) + expect(finished).to.deep.include({ + success: false, + failedUploads: ['output/bad.csv'], + errorType: 'UploadFileError', + }) + + // user-facing stderr line surfaced via outputCallback + expect(outputCalls).to.have.lengthOf(1) + const [stream, line] = outputCalls[0] + expect(stream).to.equal('stderr') + expect(line).to.include('output/bad.csv') + expect(line).to.include('duplicate_file_name') + }) + + it('lists every file in failedUploads and surfaces a single stderr line when fileUploader rejects', async function () { + const uploader = sinon.stub().rejects(new Error('network down')) + const { worker, lifecycleEvents, outputCalls } = + setupClientWithUploadTracking({ fileUploader: uploader }) + + emitRunResult(worker, 'exec-up-4', [ + { relativePath: 'a.csv', content: new TextEncoder().encode('1') }, + { relativePath: 'b.csv', content: new TextEncoder().encode('2') }, + ]) + await waitFor(() => Boolean(findFinished(lifecycleEvents))) + + const finished = findFinished(lifecycleEvents) + expect(finished).to.deep.include({ + success: false, + errorType: 'UploadFileError', + }) + expect(finished) + .to.have.property('failedUploads') + .that.has.members(['a.csv', 'b.csv']) + + expect(outputCalls).to.have.lengthOf(1) + const [stream, line] = outputCalls[0] + expect(stream).to.equal('stderr') + expect(line).to.include('network down') + }) + + it('does not invoke fileUploader when run-code-result has success: false', async function () { + const uploader = sinon.stub().resolves([]) + const { worker, lifecycleEvents } = setupClientWithUploadTracking({ + fileUploader: uploader, + }) + + emitRunResult( + worker, + 'exec-up-5', + [{ relativePath: 'a.csv', content: new TextEncoder().encode('1') }], + false + ) + await waitFor(() => Boolean(findFinished(lifecycleEvents))) + + expect(uploader.called).to.be.false + expect(findFinished(lifecycleEvents)).to.deep.include({ + success: false, + failedUploads: [], + }) + }) + + it('does not invoke fileUploader when outputFiles is empty', async function () { + const uploader = sinon.stub().resolves([]) + const { worker, lifecycleEvents } = setupClientWithUploadTracking({ + fileUploader: uploader, + }) + + emitRunResult(worker, 'exec-up-6', []) + await waitFor(() => Boolean(findFinished(lifecycleEvents))) + + expect(uploader.called).to.be.false + }) + }) + + describe('output-line forwarding', function () { + function setupClientWithOutputTracking() { + const outputCalls: Parameters[] = [] + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + onOutput: (...args) => { + outputCalls.push(args) + }, + fileUploader: fileUploaderStub, + }) + const worker = WorkerMock.instances[0] + worker.emitMessage({ type: 'listening' }) + return { client, worker, outputCalls } + } + + it('forwards stdout output-line events to the output callback', function () { + const { worker, outputCalls } = setupClientWithOutputTracking() + + worker.emitMessage({ + type: 'output-line', + stream: 'stdout', + line: 'hello!', + fileId: 'main.py', + executionId: 'exec-stdout', + }) + + expect(outputCalls).to.deep.equal([ + ['stdout', 'hello!', 'main.py', 'exec-stdout'], + ]) + }) + + it('forwards stderr output-line events to the output callback', function () { + const { worker, outputCalls } = setupClientWithOutputTracking() + + worker.emitMessage({ + type: 'output-line', + stream: 'stderr', + line: 'boom', + fileId: 'main.py', + executionId: 'exec-stderr', + }) + + expect(outputCalls).to.deep.equal([ + ['stderr', 'boom', 'main.py', 'exec-stderr'], + ]) + }) + + it('forwards info output-line events to the output callback', function () { + const { worker, outputCalls } = setupClientWithOutputTracking() + + worker.emitMessage({ + type: 'output-line', + stream: 'info', + line: 'Loading numpy from package index', + fileId: 'main.py', + executionId: 'exec-info', + }) + + expect(outputCalls).to.deep.equal([ + ['info', 'Loading numpy from package index', 'main.py', 'exec-info'], + ]) + }) + + it('preserves stream type when forwarding stdout, stderr, and info in sequence', function () { + const { worker, outputCalls } = setupClientWithOutputTracking() + + worker.emitMessage({ + type: 'output-line', + stream: 'info', + line: 'Loading package', + fileId: 'main.py', + executionId: 'exec-mixed', + }) + worker.emitMessage({ + type: 'output-line', + stream: 'stdout', + line: 'result', + fileId: 'main.py', + executionId: 'exec-mixed', + }) + worker.emitMessage({ + type: 'output-line', + stream: 'stderr', + line: 'warning', + fileId: 'main.py', + executionId: 'exec-mixed', + }) + + expect(outputCalls.map(call => call[0])).to.deep.equal([ + 'info', + 'stdout', + 'stderr', + ]) + }) + }) + + describe('reset', function () { + it('terminates the current worker and creates a new one', function () { + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + fileUploader: fileUploaderStub, + }) + const originalWorker = WorkerMock.instances[0] + originalWorker.emitMessage({ type: 'listening' }) + originalWorker.emitMessage({ type: 'loaded' }) + + client.reset() + + expect(originalWorker.terminated).to.equal(true) + expect(WorkerMock.instances).to.have.length(2) + }) + + it('sends init to the new worker once it reports listening after reset', function () { + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + fileUploader: fileUploaderStub, + }) + const originalWorker = WorkerMock.instances[0] + originalWorker.emitMessage({ type: 'listening' }) + originalWorker.emitMessage({ type: 'loaded' }) + + client.reset() + + const newWorker = WorkerMock.instances[1] + expect(newWorker.postedMessages).to.have.length(0) + + newWorker.emitMessage({ type: 'listening' }) + expect(newWorker.postedMessages).to.deep.equal([ + { + type: 'init', + baseAssetPath: BASE_ASSET_PATH, + }, + ]) + }) + + it('allows running code on the new worker after reset', function () { + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + fileUploader: fileUploaderStub, + }) + const originalWorker = WorkerMock.instances[0] + originalWorker.emitMessage({ type: 'listening' }) + originalWorker.emitMessage({ type: 'loaded' }) + + client.reset() + + const newWorker = WorkerMock.instances[1] + newWorker.emitMessage({ type: 'listening' }) + newWorker.emitMessage({ type: 'loaded' }) + + client.runCode('print("after reset")', { + fileId: 'main.py', + executionId: 'exec-5', + files: [], + }) + + const runRequest = newWorker.postedMessages.find( + (message: any) => message.type === 'run-code' + ) + expect(runRequest).to.include({ + type: 'run-code', + fileId: 'main.py', + executionId: 'exec-5', + code: 'print("after reset")', + }) + }) + + it('reset is a no-op after destroy', function () { + const client = new PyodideWorkerClient({ + baseAssetPath: BASE_ASSET_PATH, + createWorker, + fileUploader: fileUploaderStub, + }) + const originalWorker = WorkerMock.instances[0] + originalWorker.emitMessage({ type: 'listening' }) + originalWorker.emitMessage({ type: 'loaded' }) + + client.destroy() + client.reset() + + // No new worker should have been created + expect(WorkerMock.instances).to.have.length(1) + }) + }) +}) diff --git a/services/web/modules/python-runner/test/unit/editor/pyodide-worker-output-limits.spec.ts b/services/web/modules/python-runner/test/unit/editor/pyodide-worker-output-limits.spec.ts new file mode 100644 index 0000000000..5e98df55c6 --- /dev/null +++ b/services/web/modules/python-runner/test/unit/editor/pyodide-worker-output-limits.spec.ts @@ -0,0 +1,93 @@ +import { expect } from 'chai' +import { + MAX_OUTPUT_FILES, + MAX_OUTPUT_FILE_BYTES, + MAX_OUTPUT_TOTAL_BYTES, + checkOutputLimits, +} from '../../../frontend/js/components/editor/python/pyodide-worker-output-limits' + +const BYTES_PER_MB = 1024 * 1024 + +function makeFiles( + count: number, + sizePerFile: number +): { path: string; size: number }[] { + return Array.from({ length: count }, (_, i) => ({ + path: `/project/file${i}.bin`, + size: sizePerFile, + })) +} + +describe('checkOutputLimits', function () { + it('returns null when both limits are within bounds', function () { + const files = makeFiles(10, 1024) + expect(checkOutputLimits(files)).to.equal(null) + }) + + it('returns null for an empty file list', function () { + expect(checkOutputLimits([])).to.equal(null) + }) + + it('returns null at exactly the file count limit', function () { + const files = makeFiles(MAX_OUTPUT_FILES, 1024) + expect(checkOutputLimits(files)).to.equal(null) + }) + + it('returns a count violation reporting the actual file count when the file count exceeds the limit', function () { + const files = makeFiles(73, 1) + const violation = checkOutputLimits(files) + expect(violation).to.deep.equal({ + kind: 'count', + message: 'Output limit exceeded: 73 files generated (max 50)', + }) + }) + + it('returns null at exactly the per-file size limit', function () { + const files = [{ path: '/project/big.bin', size: MAX_OUTPUT_FILE_BYTES }] + expect(checkOutputLimits(files)).to.equal(null) + }) + + it('returns a single-file-size violation reporting the offending file path and size when one file exceeds the per-file limit', function () { + const files = [ + { path: '/project/small.bin', size: 1 * BYTES_PER_MB }, + { path: '/project/huge.bin', size: 80 * BYTES_PER_MB }, + ] + const violation = checkOutputLimits(files) + expect(violation).to.deep.equal({ + kind: 'single-file-size', + message: + 'Output limit exceeded: /project/huge.bin is 80MB (max 50MB per file)', + }) + }) + + it('returns a total-output-size violation when the summed size exceeds the total limit', function () { + const files = [ + { path: '/project/a.bin', size: 40 * BYTES_PER_MB }, + { path: '/project/b.bin', size: 40 * BYTES_PER_MB }, + { path: '/project/c.bin', size: 40 * BYTES_PER_MB }, + ] + const violation = checkOutputLimits(files) + expect(violation).to.not.equal(null) + expect(violation!.kind).to.equal('total-output-size') + expect(violation!.message).to.equal( + 'Output limit exceeded: 120MB total (max 100MB)' + ) + }) + + it('returns null at exactly the total size limit', function () { + const files = [ + { path: '/project/a.bin', size: 50 * BYTES_PER_MB }, + { path: '/project/b.bin', size: 50 * BYTES_PER_MB }, + ] + expect(checkOutputLimits(files)).to.equal(null) + const total = files.reduce((acc, f) => acc + f.size, 0) + expect(total).to.equal(MAX_OUTPUT_TOTAL_BYTES) + }) + + it('reports the count violation when both limits are exceeded', function () { + const files = makeFiles(MAX_OUTPUT_FILES + 10, MAX_OUTPUT_TOTAL_BYTES) + const violation = checkOutputLimits(files) + expect(violation).to.not.equal(null) + expect(violation!.kind).to.equal('count') + }) +}) diff --git a/services/web/modules/python-runner/test/unit/editor/python-runner.spec.ts b/services/web/modules/python-runner/test/unit/editor/python-runner.spec.ts new file mode 100644 index 0000000000..f3f3525866 --- /dev/null +++ b/services/web/modules/python-runner/test/unit/editor/python-runner.spec.ts @@ -0,0 +1,499 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import { + PythonRunner, + PythonRunnerState, + DEFAULT_STATE, + ExecutionContext, + type FileUploader, +} from '../../../frontend/js/components/editor/python/python-runner' +import { WorkerMock, createWorker } from './worker-mock' + +const BASE_ASSET_PATH = 'https://assets.example.test/' +const FILE_ID = 'file-1' + +function createRunner( + overrides: { + fileId?: string + getExecutionContext?: () => Promise + fileUploader?: FileUploader + } = {} +) { + const fileId = overrides.fileId ?? FILE_ID + const getExecutionContext = + overrides.getExecutionContext ?? + (() => + Promise.resolve({ + code: 'print("hello")', + files: [{ relativePath: 'main.py', content: 'print("hello")' }], + })) + + const fileUploader = overrides.fileUploader ?? sinon.stub().resolves([]) + + const runner = new PythonRunner( + fileId, + BASE_ASSET_PATH, + getExecutionContext, + createWorker, + fileUploader + ) + return runner +} + +function initAndLoad(runner: PythonRunner) { + runner.init() + const worker = WorkerMock.instances[WorkerMock.instances.length - 1] + worker.emitMessage({ type: 'listening' }) + worker.emitMessage({ type: 'loaded' }) + return worker +} + +function waitForState( + runner: PythonRunner, + predicate: (state: PythonRunnerState) => boolean +): Promise { + return new Promise(resolve => { + if (predicate(runner.getState())) { + resolve(runner.getState()) + return + } + const unsubscribe = runner.subscribe(() => { + if (predicate(runner.getState())) { + unsubscribe() + resolve(runner.getState()) + } + }) + }) +} + +describe('PythonRunner', function () { + beforeEach(function () { + WorkerMock.instances.length = 0 + }) + + describe('initial state', function () { + it('starts with default snapshot before init', function () { + const runner = createRunner() + expect(runner.getState()).to.deep.equal(DEFAULT_STATE) + }) + }) + + describe('init and lifecycle', function () { + it('transitions to loading on init', function () { + const runner = createRunner() + runner.init() + expect(runner.getState().status).to.equal('loading') + }) + + it('transitions to idle when worker reports loaded', function () { + const runner = createRunner() + initAndLoad(runner) + expect(runner.getState().status).to.equal('idle') + }) + + it('transitions to errored on loading failure', function () { + const runner = createRunner() + runner.init() + const worker = WorkerMock.instances[0] + worker.emitMessage({ type: 'listening' }) + worker.emitMessage({ + type: 'loading-failed', + error: 'network error', + }) + expect(runner.getState().status).to.equal('errored') + expect(runner.getState().error).to.equal('network error') + }) + + it('clears error on successful load after failure', function () { + const runner = createRunner() + runner.init() + const worker = WorkerMock.instances[0] + worker.emitMessage({ type: 'listening' }) + worker.emitMessage({ type: 'loaded' }) + expect(runner.getState().error).to.equal(null) + }) + + it('is a no-op if already initialized', function () { + const runner = createRunner() + runner.init() + runner.init() + expect(WorkerMock.instances).to.have.length(1) + }) + }) + + describe('run', function () { + it('transitions to running then finished', async function () { + const runner = createRunner() + const worker = initAndLoad(runner) + + await runner.run() + expect(runner.getState().status).to.equal('running') + + const runMsg = worker.postedMessages.find(m => m.type === 'run-code') + worker.emitMessage({ + type: 'run-code-result', + fileId: FILE_ID, + executionId: runMsg.executionId, + success: true, + outputs: [], + outputFiles: [], + imports: [], + }) + + await waitForState(runner, s => s.status === 'finished') + expect(runner.getState().status).to.equal('finished') + }) + + it('clears previous output on new run', async function () { + const runner = createRunner() + const worker = initAndLoad(runner) + + await runner.run() + const runMsg = worker.postedMessages.find(m => m.type === 'run-code') + worker.emitMessage({ + type: 'output-line', + stream: 'stdout', + line: 'first run output', + fileId: FILE_ID, + executionId: runMsg.executionId, + }) + worker.emitMessage({ + type: 'run-code-result', + fileId: FILE_ID, + executionId: runMsg.executionId, + success: true, + outputs: [], + outputFiles: [], + imports: [], + }) + expect(runner.getState().output).to.deep.equal([ + { stream: 'stdout', line: 'first run output' }, + ]) + + await runner.run() + expect(runner.getState().output).to.deep.equal([]) + }) + + it('is a no-op while still loading', async function () { + const runner = createRunner() + runner.init() + await runner.run() + expect(runner.getState().status).to.equal('loading') + }) + + it('is a no-op when getExecutionContext returns null', async function () { + const runner = createRunner({ + getExecutionContext: () => Promise.resolve(null), + }) + initAndLoad(runner) + + await runner.run() + expect(runner.getState().status).to.equal('idle') + }) + + it('transitions to errored when getExecutionContext rejects', async function () { + const runner = createRunner({ + getExecutionContext: () => Promise.reject(new Error('network failure')), + }) + initAndLoad(runner) + + await runner.run() + expect(runner.getState().status).to.equal('errored') + expect(runner.getState().error).to.equal('network failure') + }) + }) + + describe('files-saved toast', function () { + let toastEvents: CustomEvent[] + let toastListener: (event: Event) => void + + beforeEach(function () { + toastEvents = [] + toastListener = event => { + toastEvents.push(event as CustomEvent) + } + window.addEventListener('ide:show-toast', toastListener) + }) + + afterEach(function () { + window.removeEventListener('ide:show-toast', toastListener) + }) + + it('dispatches a files-saved toast with successfully uploaded paths', async function () { + const runner = createRunner() + const worker = initAndLoad(runner) + + await runner.run() + const runMsg = worker.postedMessages.find(m => m.type === 'run-code') + worker.emitMessage({ + type: 'run-code-result', + fileId: FILE_ID, + executionId: runMsg.executionId, + success: true, + outputs: ['/project/foo.txt', '/project/bar.csv'], + outputFiles: [], + imports: [], + failedUploads: [], + }) + + await waitForState(runner, s => s.status === 'finished') + + expect(toastEvents).to.have.length(1) + expect(toastEvents[0].detail).to.deep.equal({ + key: 'python:files-saved', + paths: ['foo.txt', 'bar.csv'], + }) + }) + + it('excludes failed uploads from the toast', async function () { + const fileUploader = sinon.stub().resolves([ + { status: 'success', name: 'foo.txt', relativePath: 'foo.txt' }, + { + status: 'error', + name: 'bar.csv', + relativePath: 'bar.csv', + error: 'boom', + }, + ]) + const runner = createRunner({ fileUploader }) + const worker = initAndLoad(runner) + + await runner.run() + const runMsg = worker.postedMessages.find(m => m.type === 'run-code') + worker.emitMessage({ + type: 'run-code-result', + fileId: FILE_ID, + executionId: runMsg.executionId, + success: true, + outputs: ['/project/foo.txt', '/project/bar.csv'], + outputFiles: [ + { relativePath: 'foo.txt', content: new Uint8Array() }, + { relativePath: 'bar.csv', content: new Uint8Array() }, + ], + imports: [], + }) + + await waitForState(runner, s => s.status === 'finished') + + expect(toastEvents).to.have.length(1) + expect(toastEvents[0].detail).to.deep.equal({ + key: 'python:files-saved', + paths: ['foo.txt'], + }) + }) + + it('does not dispatch a toast when no outputs were written', async function () { + const runner = createRunner() + const worker = initAndLoad(runner) + + await runner.run() + const runMsg = worker.postedMessages.find(m => m.type === 'run-code') + worker.emitMessage({ + type: 'run-code-result', + fileId: FILE_ID, + executionId: runMsg.executionId, + success: true, + outputs: [], + outputFiles: [], + imports: [], + failedUploads: [], + }) + + await waitForState(runner, s => s.status === 'finished') + + expect(toastEvents).to.have.length(0) + }) + + it('does not dispatch a toast when every output failed to upload', async function () { + const fileUploader = sinon.stub().resolves([ + { + status: 'error', + name: 'foo.txt', + relativePath: 'foo.txt', + error: 'boom', + }, + ]) + const runner = createRunner({ fileUploader }) + const worker = initAndLoad(runner) + + await runner.run() + const runMsg = worker.postedMessages.find(m => m.type === 'run-code') + worker.emitMessage({ + type: 'run-code-result', + fileId: FILE_ID, + executionId: runMsg.executionId, + success: true, + outputs: ['/project/foo.txt'], + outputFiles: [{ relativePath: 'foo.txt', content: new Uint8Array() }], + imports: [], + }) + + await waitForState(runner, s => s.status === 'finished') + + expect(toastEvents).to.have.length(0) + }) + }) + + describe('output', function () { + it('accumulates output lines for the matching file', async function () { + const runner = createRunner() + const worker = initAndLoad(runner) + + await runner.run() + const runMsg = worker.postedMessages.find(m => m.type === 'run-code') + + worker.emitMessage({ + type: 'output-line', + stream: 'stdout', + line: 'line 1', + fileId: FILE_ID, + executionId: runMsg.executionId, + }) + worker.emitMessage({ + type: 'output-line', + stream: 'stderr', + line: 'line 2', + fileId: FILE_ID, + executionId: runMsg.executionId, + }) + + expect(runner.getState().output).to.deep.equal([ + { stream: 'stdout', line: 'line 1' }, + { stream: 'stderr', line: 'line 2' }, + ]) + }) + + it('ignores output for a different fileId', async function () { + const runner = createRunner() + const worker = initAndLoad(runner) + + await runner.run() + const runMsg = worker.postedMessages.find(m => m.type === 'run-code') + + worker.emitMessage({ + type: 'output-line', + stream: 'stdout', + line: 'other file output', + fileId: 'different-file', + executionId: runMsg.executionId, + }) + + expect(runner.getState().output).to.deep.equal([]) + }) + + it('ignores output for a stale executionId', async function () { + const runner = createRunner() + const worker = initAndLoad(runner) + + await runner.run() + + worker.emitMessage({ + type: 'output-line', + stream: 'stdout', + line: 'stale output', + fileId: FILE_ID, + executionId: 'old-execution-id', + }) + + expect(runner.getState().output).to.deep.equal([]) + }) + + it('caps output at 100 lines', async function () { + const runner = createRunner() + const worker = initAndLoad(runner) + + await runner.run() + const runMsg = worker.postedMessages.find(m => m.type === 'run-code') + + for (let i = 0; i < 110; i++) { + worker.emitMessage({ + type: 'output-line', + stream: 'stdout', + line: `line ${i}`, + fileId: FILE_ID, + executionId: runMsg.executionId, + }) + } + + const output = runner.getState().output + expect(output).to.have.length(100) + expect(output[0]).to.deep.equal({ stream: 'stdout', line: 'line 10' }) + expect(output[99]).to.deep.equal({ stream: 'stdout', line: 'line 109' }) + }) + }) + + describe('interrupt', function () { + it('appends interrupted message and transitions to loading when running', async function () { + const runner = createRunner() + const worker = initAndLoad(runner) + + await runner.run() + const runMsg = worker.postedMessages.find(m => m.type === 'run-code') + worker.emitMessage({ + type: 'output-line', + stream: 'stdout', + line: 'partial output', + fileId: FILE_ID, + executionId: runMsg.executionId, + }) + + runner.interrupt() + + expect(runner.getState().status).to.equal('loading') + expect(runner.getState().output).to.deep.equal([ + { stream: 'stdout', line: 'partial output' }, + { stream: 'info', line: 'Execution interrupted' }, + ]) + }) + + it('does not append interrupted message when not running', function () { + const runner = createRunner() + initAndLoad(runner) + + runner.interrupt() + + expect(runner.getState().status).to.equal('loading') + expect(runner.getState().output).to.deep.equal([]) + }) + }) + + describe('subscribe', function () { + it('notifies listeners on state changes', function () { + const runner = createRunner() + const listener = sinon.stub() + + runner.subscribe(listener) + initAndLoad(runner) + + expect(listener.callCount).to.be.greaterThan(0) + }) + + it('stops notifying after unsubscribe', function () { + const runner = createRunner() + const listener = sinon.stub() + + const unsubscribe = runner.subscribe(listener) + runner.init() + const countAfterInit = listener.callCount + + unsubscribe() + const worker = WorkerMock.instances[0] + worker.emitMessage({ type: 'listening' }) + worker.emitMessage({ type: 'loaded' }) + + expect(listener.callCount).to.equal(countAfterInit) + }) + }) + + describe('destroy', function () { + it('terminates the worker', function () { + const runner = createRunner() + initAndLoad(runner) + + runner.destroy() + + const worker = WorkerMock.instances[0] + expect(worker.terminated).to.equal(true) + }) + }) +}) diff --git a/services/web/modules/python-runner/test/unit/editor/worker-mock.ts b/services/web/modules/python-runner/test/unit/editor/worker-mock.ts new file mode 100644 index 0000000000..b14697f0a4 --- /dev/null +++ b/services/web/modules/python-runner/test/unit/editor/worker-mock.ts @@ -0,0 +1,35 @@ +type WorkerMessageListener = (event: MessageEvent) => void + +export class WorkerMock { + static instances: WorkerMock[] = [] + + readonly postedMessages: any[] = [] + terminated = false + private messageListeners: WorkerMessageListener[] = [] + + constructor() { + WorkerMock.instances.push(this) + } + + addEventListener(type: string, listener: WorkerMessageListener) { + if (type === 'message') { + this.messageListeners.push(listener) + } + } + + postMessage(message: unknown) { + this.postedMessages.push(message) + } + + terminate() { + this.terminated = true + } + + emitMessage(message: unknown) { + for (const listener of this.messageListeners) { + listener({ data: message, target: this } as unknown as MessageEvent) + } + } +} + +export const createWorker = () => new WorkerMock() as unknown as Worker diff --git a/services/web/scripts/fetch-pyodide-packages.mjs b/services/web/scripts/fetch-pyodide-packages.mjs new file mode 100644 index 0000000000..dc10bc856f --- /dev/null +++ b/services/web/scripts/fetch-pyodide-packages.mjs @@ -0,0 +1,136 @@ +/* eslint-disable @overleaf/require-script-runner */ +// This script doesn't work with ScriptRunner because it is run during the build process. +import { createReadStream, createWriteStream } from 'node:fs' +import { mkdir, readdir, rm, stat, writeFile } from 'node:fs/promises' +import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' +import { execFile } from 'node:child_process' +import { createHash } from 'node:crypto' +import { promisify } from 'node:util' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const execFileAsync = promisify(execFile) + +const SERVICE_WEB_DIR = path.resolve(fileURLToPath(import.meta.url), '../..') + +// Pinned pyodide release tarball. Keep PYODIDE_VERSION in sync with the +// "pyodide" entry in services/web/package.json. When bumping, update both +// PYODIDE_VERSION and EXPECTED_SHA256 together; fetch the hash via: +// curl -sL https://api.github.com/repos/pyodide/pyodide/releases/tags/ \ +// | jq -r '.assets[] | select(.name=="pyodide-.tar.bz2") | .digest' +// (strip the "sha256:" prefix). Cross-check by downloading the tarball and +// running `shasum -a 256 pyodide-.tar.bz2`. +const PYODIDE_VERSION = '0.29.3' +const EXPECTED_SHA256 = + '458e8ddbcbb6e21037d3237cd5c5146c451765bc738dfa2249ff34c5140331e4' +const TARGET_DIR = path.join( + SERVICE_WEB_DIR, + 'public/js/libs/pyodide', + PYODIDE_VERSION +) +const TARBALL_NAME = `pyodide-${PYODIDE_VERSION}.tar.bz2` +const RELEASE_URL = `https://github.com/pyodide/pyodide/releases/download/${PYODIDE_VERSION}/${TARBALL_NAME}` +const COMPLETE_MARKER = path.join(TARGET_DIR, '.fetch-complete') + +async function download(url, dest) { + console.log(`Downloading ${url}`) + const res = await fetch(url, { redirect: 'follow' }) + if (!res.ok) { + throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`) + } + await pipeline(Readable.fromWeb(res.body), createWriteStream(dest)) +} + +async function sha256(file) { + const hash = createHash('sha256') + await pipeline(createReadStream(file), hash) + return hash.digest('hex') +} + +// The version subdir only needs what pyodide fetches via packageBaseUrl +// (wheels, their .metadata sidecars, and lib*.zip shared libraries). Skip +// everything else: +// - core runtime (pyodide.mjs / asm / stdlib / lock) lives one level up, +// copied from the npm package by webpack CopyPlugin. +// - *-tests.tar / test-*.zip: per-package test fixtures and pyodide's own +// test packages, not used at runtime. +// - console*.html, python / python.exe / python.bat / python_cli_entry.mjs, +// README.md: REPL UI, CLI shims, and docs. +const TAR_EXCLUDES = [ + 'pyodide.mjs', + 'pyodide.asm.js', + 'pyodide.asm.wasm', + 'python_stdlib.zip', + 'pyodide-lock.json', + '*-tests.tar', + 'test-*.zip', + 'console*.html', + 'python', + 'python.exe', + 'python.bat', + 'python_cli_entry.mjs', + 'README.md', +] + +async function extract(tarball, targetDir) { + console.log(`Extracting ${path.basename(tarball)}`) + // Tarball contains a top-level pyodide/ folder; strip it so contents land + // directly in targetDir. + await execFileAsync('tar', [ + '-xjf', + tarball, + '-C', + targetDir, + '--strip-components=1', + ...TAR_EXCLUDES.map(p => `--exclude=${p}`), + ]) +} + +async function main() { + try { + await stat(COMPLETE_MARKER) + console.log(`Pyodide ${PYODIDE_VERSION} already present at ${TARGET_DIR}`) + return + } catch (err) { + if (err.code !== 'ENOENT') throw err + } + + // A prior run may have left a partial install without the marker; wipe it + // so extraction starts from a clean directory. + await rm(TARGET_DIR, { recursive: true, force: true }) + await mkdir(TARGET_DIR, { recursive: true }) + + const tarballPath = path.join(TARGET_DIR, TARBALL_NAME) + try { + await download(RELEASE_URL, tarballPath) + const actual = await sha256(tarballPath) + if (actual !== EXPECTED_SHA256) { + throw new Error( + `SHA-256 mismatch for ${TARBALL_NAME}: expected ${EXPECTED_SHA256}, got ${actual}` + ) + } + await extract(tarballPath, TARGET_DIR) + await rm(tarballPath, { force: true }) + + const extracted = await readdir(TARGET_DIR) + if (!extracted.some(name => name.endsWith('.whl'))) { + throw new Error( + `Extraction did not produce any wheels under ${TARGET_DIR}` + ) + } + + await writeFile(COMPLETE_MARKER, '') + } catch (err) { + // Leave no partial install behind, so the next run starts clean. + await rm(TARGET_DIR, { recursive: true, force: true }) + throw err + } + + console.log(`Pyodide ${PYODIDE_VERSION} ready at ${TARGET_DIR}`) +} + +main().catch(err => { + console.error(err) + process.exit(1) +})