agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-15 14:43:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #34363 from overleaf/cd-migrate-python-runner-to-modules

Browse Source 
Migrate the Python script runner into a closed python-runner module

GitOrigin-RevId: ac22f45bcf744c693591076f0b41e8d52249dcbf
This commit is contained in:
Chris Dryden
2026-06-10 08:06:00 +00:00
committed by Copybot
parent dceb766db0
commit b90a8500d5
18 changed files with 15 additions and 3297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/web/config/settings.defaults.js
+1
View File
@@ -1041,6 +1041,7 @@ module.exports = {
rootContextProviders: [], rootContextProviders: [],
mainEditorLayoutModals: [], mainEditorLayoutModals: [],
mainEditorLayoutPanels: [], mainEditorLayoutPanels: [],
pythonRunner: [],
langFeedbackLinkingWidgets: [], langFeedbackLinkingWidgets: [],
labsExperiments: [], labsExperiments: [],
integrationLinkingWidgets: [], integrationLinkingWidgets: [],
services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts
-225
View File
@@ -1,225 +0,0 @@
import path from 'path-browserify'
import type {
ExecutionErrorType,
OutputStream,
ProjectFileData,
PyodideWorkerRequest,
PyodideWorkerResponse,
} from './pyodide-worker-messages'
import type { BatchUploadItem } from '@/infrastructure/batch-file-uploader'
import type { FileUploader } from './python-runner'
export type OutputCallback = (
stream: OutputStream,
line: string,
fileId: string,
executionId: string
) => void
export type LifecycleCallback = (
event:
| { type: 'loaded' }
| { type: 'loading-failed'; error: string }
| {
type: 'run-finished'
fileId: string
executionId: string
success: boolean
outputs: string[]
failedUploads: string[]
imports: string[]
errorType?: ExecutionErrorType
}
) => void
export class PyodideWorkerClient {
private worker: Worker
private baseAssetPath: string
private createWorker: () => Worker
private listening = false
private destroyed = false
private loadingError: string | null = null
private pendingMessages: PyodideWorkerRequest[] = []
private outputCallback: OutputCallback | null
private lifecycleCallback: LifecycleCallback | null
private fileUploader: FileUploader
constructor(options: {
baseAssetPath: string
createWorker: () => Worker
onOutput?: OutputCallback
onLifecycle?: LifecycleCallback
fileUploader: FileUploader
}) {
this.baseAssetPath = options.baseAssetPath
this.createWorker = options.createWorker
this.outputCallback = options.onOutput ?? null
this.lifecycleCallback = options.onLifecycle ?? null
this.fileUploader = options.fileUploader
this.worker = this.createWorker()
this.worker.addEventListener('message', this.receive)
this.queueMessage({
type: 'init',
baseAssetPath: this.baseAssetPath,
})
}
runCode(
code: string,
options: { fileId: string; executionId: string; files: ProjectFileData[] }
): void {
if (this.destroyed) {
throw new Error('Pyodide worker client has been destroyed')
}
if (this.loadingError) {
throw new Error(this.loadingError)
}
this.queueMessage({
type: 'run-code',
code,
fileId: options.fileId,
executionId: options.executionId,
files: options.files,
})
}
reset(): void {
if (this.destroyed) {
return
}
// Terminate the current worker immediately
this.worker.terminate()
this.pendingMessages.length = 0
// Reset state for the new worker
this.listening = false
this.loadingError = null
// Create a fresh worker and re-initialize Pyodide
this.worker = this.createWorker()
this.worker.addEventListener('message', this.receive)
this.queueMessage({
type: 'init',
baseAssetPath: this.baseAssetPath,
})
}
destroy() {
if (this.destroyed) {
return
}
this.destroyed = true
this.pendingMessages.length = 0
this.worker.terminate()
}
private queueMessage(message: PyodideWorkerRequest) {
if (this.listening) {
this.worker.postMessage(message)
} else {
this.pendingMessages.push(message)
}
}
private receive = async (event: MessageEvent<PyodideWorkerResponse>) => {
// Discard messages from a previously terminated worker
if (event.target !== this.worker) {
return
}
const response = event.data
switch (response.type) {
case 'listening':
this.listening = true
for (const message of this.pendingMessages) {
this.worker.postMessage(message)
}
this.pendingMessages.length = 0
return
case 'loaded':
this.lifecycleCallback?.({ type: 'loaded' })
return
case 'loading-failed':
this.loadingError = response.error
this.pendingMessages.length = 0
this.lifecycleCallback?.({
type: 'loading-failed',
error: response.error,
})
return
case 'output-line':
this.outputCallback?.(
response.stream,
response.line,
response.fileId,
response.executionId
)
return
case 'run-code-result': {
let success = response.success
const failedUploads: string[] = []
if (success && response.outputFiles.length > 0) {
const items: BatchUploadItem[] = response.outputFiles.map(file => ({
file: new Blob([file.content as Uint8Array<ArrayBuffer>]),
name: path.basename(file.relativePath),
relativePath: file.relativePath,
}))
try {
const results = await this.fileUploader(items)
for (const result of results) {
if (result.status === 'error') {
failedUploads.push(result.relativePath!)
this.outputCallback?.(
'stderr',
`Failed to upload output file ${result.relativePath!}: ${result.error}`,
response.fileId,
response.executionId
)
}
}
if (failedUploads.length > 0) {
success = false
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err)
this.outputCallback?.(
'stderr',
`Failed to upload output files: ${message}`,
response.fileId,
response.executionId
)
failedUploads.push(...items.map(item => item.relativePath!))
success = false
}
}
const errorType =
failedUploads.length > 0 ? 'UploadFileError' : response.errorType
this.lifecycleCallback?.({
type: 'run-finished',
fileId: response.fileId,
executionId: response.executionId,
success,
outputs: response.outputs,
failedUploads,
imports: response.imports,
errorType,
})
}
}
}
}
services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts
-72
View File
@@ -1,72 +0,0 @@
export type OutputStream = 'stdout' | 'stderr' | 'info'
export type ProjectFileData = {
relativePath: string
content: string
}
export type OutputFileData = {
relativePath: string
content: Uint8Array
}
// Main thread -> Worker messages
export type InitRequest = {
type: 'init'
baseAssetPath: string
}
export type RunCodeRequest = {
type: 'run-code'
fileId: string
executionId: string
code: string
files: ProjectFileData[]
}
export type PyodideWorkerRequest = InitRequest | RunCodeRequest
// Worker -> Main thread lifecycle and streaming events
export type ListeningEvent = { type: 'listening' }
export type LoadedEvent = { type: 'loaded' }
export type LoadingFailedEvent = { type: 'loading-failed'; error: string }
export type OutputLineEvent = {
type: 'output-line'
stream: OutputStream
line: string
fileId: string
executionId: string
}
export type PyodideWorkerEvent =
| ListeningEvent
| LoadedEvent
| LoadingFailedEvent
| OutputLineEvent
// Worker -> Main thread ID responses
export type ExecutionErrorType =
| 'SyntaxError'
| 'ModuleNotFoundError'
| 'OutputLimitExceeded'
| 'UploadFileError'
| 'generic'
export type ExecutionResult = 'success' | 'error'
export type RunCodeResult = {
type: 'run-code-result'
fileId: string
executionId: string
success: boolean
outputs: string[]
outputFiles: OutputFileData[]
imports: string[]
errorType?: ExecutionErrorType
}
export type PyodideWorkerResponse = PyodideWorkerEvent | RunCodeResult
services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-output-limits.ts
-53
View File
@@ -1,53 +0,0 @@
const BYTES_PER_MB = 1024 * 1024
export const MAX_OUTPUT_FILES = 50
export const MAX_OUTPUT_TOTAL_BYTES = 100 * BYTES_PER_MB
export const MAX_OUTPUT_FILE_BYTES = 50 * BYTES_PER_MB
export type OutputLimitViolation = {
kind: 'count' | 'total-output-size' | 'single-file-size'
message: string
}
export function checkOutputCount(count: number): OutputLimitViolation | null {
if (count > MAX_OUTPUT_FILES) {
return {
kind: 'count',
message: `Output limit exceeded: ${count} files generated (max ${MAX_OUTPUT_FILES})`,
}
}
return null
}
export function checkOutputLimits(
files: { path: string; size: number }[]
): OutputLimitViolation | null {
const countViolation = checkOutputCount(files.length)
if (countViolation) {
return countViolation
}
let totalBytes = 0
for (const file of files) {
if (file.size > MAX_OUTPUT_FILE_BYTES) {
const fileMB = Math.ceil(file.size / BYTES_PER_MB)
const maxMB = MAX_OUTPUT_FILE_BYTES / BYTES_PER_MB
return {
kind: 'single-file-size',
message: `Output limit exceeded: ${file.path} is ${fileMB}MB (max ${maxMB}MB per file)`,
}
}
totalBytes += file.size
}
if (totalBytes > MAX_OUTPUT_TOTAL_BYTES) {
const totalMB = Math.ceil(totalBytes / BYTES_PER_MB)
const maxMB = MAX_OUTPUT_TOTAL_BYTES / BYTES_PER_MB
return {
kind: 'total-output-size',
message: `Output limit exceeded: ${totalMB}MB total (max ${maxMB}MB)`,
}
}
return null
}
services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts
-309
View File
@@ -1,309 +0,0 @@
/// <reference lib="webworker" />
import path from 'path-browserify'
import type { PyodideInterface } from 'pyodide'
import type {
ExecutionErrorType,
OutputFileData,
InitRequest,
ProjectFileData,
PyodideWorkerRequest,
RunCodeRequest,
} from './pyodide-worker-messages'
import {
checkOutputCount,
checkOutputLimits,
} from './pyodide-worker-output-limits'
type PyodideFS = PyodideInterface['FS']
type PyodideModule = typeof import('pyodide')
const PROJECT_FS_ROOT = '/project'
const PROJECT_FS_PREFIX = `${PROJECT_FS_ROOT}/`
const PYODIDE_INDEX_PATH = 'js/libs/pyodide/'
function classifyErrorType(errorMessage: string): ExecutionErrorType {
if (errorMessage.includes('ModuleNotFoundError')) {
return 'ModuleNotFoundError'
}
if (errorMessage.includes('SyntaxError')) {
return 'SyntaxError'
}
return 'generic'
}
function moduleNotFoundHelpMessage(): string {
return (
"Note: Only Pyodide's built-in packages are available in the browser. " +
'Packages installed via pip cannot be used here. ' +
'See https://pyodide.org/en/stable/usage/packages-in-pyodide.html ' +
'for the list of supported packages.'
)
}
function ensureDirectoryExists(fs: PyodideFS, filePath: string) {
const directory = path.dirname(filePath)
if (directory === '.' || directory === '/') {
return
}
let currentPath = directory.startsWith('/') ? '/' : ''
for (const part of directory.split('/').filter(Boolean)) {
currentPath = path.posix.join(currentPath, part)
try {
const analysis = fs.analyzePath(currentPath)
if (!analysis.exists) {
fs.mkdir(currentPath)
}
} catch {
// Ignore failures when a directory already exists.
}
}
}
function syncProjectFiles(fs: PyodideFS, files: ProjectFileData[]) {
for (const file of files) {
const runtimePath = path.posix.join(
PROJECT_FS_ROOT,
path.posix.normalize(file.relativePath)
)
ensureDirectoryExists(fs, runtimePath)
fs.writeFile(runtimePath, file.content)
}
fs.chdir(PROJECT_FS_ROOT)
}
let pyodideModule: PyodideModule | null = null
let pyodideIndexUrl: string | undefined
async function loadPyodideModule(pyodideIndexUrl: string) {
const runtimeModuleUrl = `${pyodideIndexUrl}pyodide.mjs`
try {
return (await import(
/* webpackIgnore: true */ runtimeModuleUrl
)) as PyodideModule
} catch (loadError) {
const loadErrorMessage =
loadError instanceof Error ? loadError.message : String(loadError)
throw new Error(
`Unable to load Pyodide module from ${runtimeModuleUrl}. Original error: ${loadErrorMessage}`
)
}
}
async function handleInit(msg: InitRequest) {
pyodideIndexUrl = new URL(PYODIDE_INDEX_PATH, msg.baseAssetPath).toString()
try {
pyodideModule = await loadPyodideModule(pyodideIndexUrl)
self.postMessage({ type: 'loaded' })
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error)
console.error('Pyodide initialization failed', error)
self.postMessage({
type: 'loading-failed',
error: errorMessage,
})
}
}
async function handleRunCode(msg: RunCodeRequest) {
const { fileId, executionId } = msg
const writtenPaths = new Set<string>()
const readPaths = new Set<string>()
const computeImports = () =>
[...readPaths].filter(path => !writtenPaths.has(path))
const postFailure = (
stream: 'stderr' | 'info',
line: string,
errorType: ExecutionErrorType = 'generic'
) => {
self.postMessage({
type: 'output-line',
stream,
line,
fileId,
executionId,
})
self.postMessage({
type: 'run-code-result',
fileId,
executionId,
success: false,
outputs: [],
outputFiles: [],
imports: computeImports(),
errorType,
})
}
if (!pyodideModule || !pyodideIndexUrl) {
postFailure('stderr', 'Pyodide is not initialized')
return
}
const instance = await pyodideModule.loadPyodide({
env: { MPLBACKEND: 'Agg' },
packageBaseUrl: `${pyodideIndexUrl}${pyodideModule.version}/`,
})
instance.setStdout({
batched: (line: string) => {
self.postMessage({
type: 'output-line',
stream: 'stdout',
line,
fileId,
executionId,
})
},
})
instance.setStderr({
batched: (line: string) => {
self.postMessage({
type: 'output-line',
stream: 'stderr',
line,
fileId,
executionId,
})
},
})
const fs = instance.FS
const originalWrite = fs.write as PyodideFS['write']
const originalRead = fs.read as PyodideFS['read']
let runError: unknown = null
try {
if (msg.files.length > 0) {
syncProjectFiles(fs, msg.files)
}
fs.write = ((...args: Parameters<PyodideFS['write']>) => {
const [stream] = args
// Only surface writes to the synced project workspace, not Pyodide internals.
if (
typeof stream?.path === 'string' &&
stream.path.startsWith(PROJECT_FS_PREFIX)
) {
writtenPaths.add(stream.path)
}
return originalWrite.call(fs, ...args)
}) as PyodideFS['write']
fs.read = ((...args: Parameters<PyodideFS['read']>) => {
const [stream] = args
if (
typeof stream?.path === 'string' &&
stream.path.startsWith(PROJECT_FS_PREFIX)
) {
readPaths.add(stream.path)
}
return originalRead.call(fs, ...args)
}) as PyodideFS['read']
await instance.loadPackagesFromImports(msg.code)
const result = await instance.runPythonAsync(msg.code)
if (result !== undefined) {
self.postMessage({
type: 'output-line',
stream: 'stdout',
line: String(result),
fileId,
executionId,
})
}
} catch (e) {
runError = e
}
fs.write = originalWrite
fs.read = originalRead
const paths = [...writtenPaths]
if (runError) {
const errorMessage =
runError instanceof Error ? runError.message : String(runError)
const errorType = classifyErrorType(errorMessage)
const fullMessage =
errorType === 'ModuleNotFoundError'
? `${errorMessage}\n${moduleNotFoundHelpMessage()}`
: errorMessage
postFailure('stderr', fullMessage, errorType)
return
}
const countViolation = checkOutputCount(paths.length)
if (countViolation) {
postFailure('info', countViolation.message, 'OutputLimitExceeded')
return
}
const filesWithSizes: { path: string; size: number }[] = []
for (const writtenPath of paths) {
try {
filesWithSizes.push({
path: writtenPath,
size: fs.stat(writtenPath).size,
})
} catch {
// A script can write a file and later delete or rename it before the run
// finishes; fs.stat would then throw and we'd never post a
// run-code-result, leaving the UI stuck. Skip paths we can't stat.
}
}
const sizeViolation = checkOutputLimits(filesWithSizes)
if (sizeViolation) {
postFailure('info', sizeViolation.message, 'OutputLimitExceeded')
return
}
const outputFiles: OutputFileData[] = []
const transferables: Transferable[] = []
for (const { path: writtenPath } of filesWithSizes) {
const content = fs.readFile(writtenPath)
const relativePath = writtenPath.slice(PROJECT_FS_PREFIX.length)
outputFiles.push({ relativePath, content })
if (content.buffer instanceof ArrayBuffer) {
transferables.push(content.buffer)
}
}
// The transferables moves ownership of each ArrayBuffer to the main thread
// instead of structured-cloning it. The buffers are already referenced from
// outputFiles.content; listing them here just swaps copy for move, so file
// contents travel through once rather than being allocated on both sides.
self.postMessage(
{
type: 'run-code-result',
fileId,
executionId,
success: true,
outputs: filesWithSizes.map(f => f.path),
outputFiles,
imports: computeImports(),
},
transferables
)
}
self.addEventListener('message', async event => {
const msg = event.data as PyodideWorkerRequest
switch (msg.type) {
case 'init':
await handleInit(msg)
break
case 'run-code':
await handleRunCode(msg)
break
}
})
self.postMessage({ type: 'listening' })
services/web/frontend/js/features/ide-react/components/editor/python/python-output-pane.tsx
-113
View File
@@ -1,113 +0,0 @@
import { useEffect, useMemo, useSyncExternalStore } from 'react'
import { useTranslation } from 'react-i18next'
import classNames from 'classnames'
import OLButton from '@/shared/components/ol/ol-button'
import OLButtonToolbar from '@/shared/components/ol/ol-button-toolbar'
import MaterialIcon from '@/shared/components/material-icon'
import SplitTestBadge from '@/shared/components/split-test-badge'
import { sendMB } from '@/infrastructure/event-tracking'
import { useEditorOpenDocContext } from '@/features/ide-react/context/editor-open-doc-context'
import { usePythonExecutionContext } from '@/features/ide-react/context/python-execution-context'
import { DEFAULT_STATE } from './python-runner'
const emptySubscribe = () => () => {}
const getDefaultState = () => DEFAULT_STATE
export default function PythonOutputPane() {
const { t } = useTranslation()
const { currentDocumentId, openDocName } = useEditorOpenDocContext()
const { getPythonRunner } = usePythonExecutionContext()
const pythonRunner = useMemo(
() => (currentDocumentId ? getPythonRunner(currentDocumentId) : null),
[currentDocumentId, getPythonRunner]
)
useEffect(() => {
if (!currentDocumentId || !openDocName) {
return
}
sendMB('script-runner-opened', {
fileName: openDocName,
fileExtension: 'py',
editorMode: 'code',
})
}, [currentDocumentId, openDocName])
const { output, error, status } = useSyncExternalStore(
pythonRunner ? pythonRunner.subscribe : emptySubscribe,
pythonRunner ? pythonRunner.getState : getDefaultState
)
if (!pythonRunner) {
return null
}
return (
<div className="ide-redesign-python-output-pane">
<OLButtonToolbar className="ide-redesign-python-output-pane-toolbar">
<div className="ide-redesign-python-output-pane-toolbar-left">
<div
className={classNames(
'ide-redesign-python-output-pane-run-button-wrapper',
{
'compile-button-group-running': status === 'running',
}
)}
>
<OLButton
onClick={() => {
if (status === 'running') {
pythonRunner.interrupt()
} else {
pythonRunner.run()
}
}}
variant={status === 'running' ? 'danger' : 'primary'}
className="align-items-center py-0 px-3"
disabled={status === 'loading'}
aria-label={
status === 'running'
? t('stop_python_execution')
: t('run_python_code')
}
>
{status === 'running' ? t('stop') : t('run')}
<MaterialIcon
type={status === 'running' ? 'stop' : 'play_arrow'}
className="ml-2"
/>
</OLButton>
</div>
<SplitTestBadge
splitTestName="overleaf-code"
displayOnVariants={['enabled']}
/>
</div>
</OLButtonToolbar>
<div className="ide-redesign-python-output-pane-body">
{status === 'loading' && (
<div className="ide-redesign-python-output-pane-placeholder">
{t('loading_python_runtime')}
</div>
)}
{status !== 'loading' && !error && output.length === 0 && (
<div className="ide-redesign-python-output-pane-placeholder">
{t('run_current_script_to_see_output')}
</div>
)}
{error && (
<div className="ide-redesign-python-output-pane-error">{error}</div>
)}
{output.map((entry, index) => (
<div
className={`ide-redesign-python-output-pane-line ide-redesign-python-output-pane-line-${entry.stream}`}
key={index}
>
{entry.line}
</div>
))}
</div>
</div>
)
}
services/web/frontend/js/features/ide-react/components/editor/python/python-output-toasts.tsx
-48
View File
@@ -1,48 +0,0 @@
import { GlobalToastGeneratorEntry } from '@/features/ide-react/components/global-toasts'
import { useTranslation } from 'react-i18next'
const stripProjectPrefix = (path: string) => path.replace(/^\/project\/?/, '')
const PythonFilesSavedToast = ({ paths }: { paths: string[] }) => {
const { t } = useTranslation()
if (paths.length === 1) {
return (
<span>
{t('x_saved_to_your_project', {
fileName: stripProjectPrefix(paths[0]),
})}
</span>
)
}
return (
<span>{t('x_files_saved_to_your_project', { count: paths.length })}</span>
)
}
const isStringArray = (value: unknown): value is string[] =>
Array.isArray(value) && value.every(v => typeof v === 'string')
const generators: GlobalToastGeneratorEntry[] = [
{
key: 'python:files-saved',
generator: ({ paths }) => ({
content: (
<PythonFilesSavedToast paths={isStringArray(paths) ? paths : []} />
),
type: 'success',
autoHide: true,
delay: 5000,
isDismissible: true,
}),
},
]
export default generators
export const showPythonFilesSavedToast = (paths: string[]) => {
window.dispatchEvent(
new CustomEvent('ide:show-toast', {
detail: { key: 'python:files-saved', paths },
})
)
}
services/web/frontend/js/features/ide-react/components/editor/python/python-runner.ts
-305
View File
@@ -1,305 +0,0 @@
// Per-file Python execution manager. Each PythonRunner owns a PyodideWorkerClient
// and exposes a subscribe/getState API for use with useSyncExternalStore,
// so React components can reactively read execution status and output.
import path from 'path-browserify'
import { v4 as uuid } from 'uuid'
import { debugConsole } from '@/utils/debugging'
import { sendMB } from '@/infrastructure/event-tracking'
import { PyodideWorkerClient } from './pyodide-worker-client'
import { showPythonFilesSavedToast } from './python-output-toasts'
import type { OutputStream } from './pyodide-worker-messages'
import type {
BatchUploadItem,
UploadResult,
} from '@/infrastructure/batch-file-uploader'
export type FileUploader = (items: BatchUploadItem[]) => Promise<UploadResult[]>
const MAX_OUTPUT_LINES = 100
const PROJECT_FS_PREFIX = '/project/'
function stripProjectFsPrefix(p: string): string {
return p.startsWith(PROJECT_FS_PREFIX) ? p.slice(PROJECT_FS_PREFIX.length) : p
}
export type ExecutionStatus =
| 'loading'
| 'idle'
| 'running'
| 'finished'
| 'errored'
export type ExecutionContext = {
code: string
files: { relativePath: string; content: string }[]
}
type Listener = () => void
export type OutputLine = {
stream: OutputStream
line: string
}
export type PythonRunnerState = {
output: OutputLine[]
status: ExecutionStatus
error: string | null
}
export const DEFAULT_STATE: PythonRunnerState = {
output: [],
status: 'loading',
error: null,
}
export class PythonRunner {
readonly fileId: string
private client: PyodideWorkerClient | null = null
private readonly baseAssetPath: string
private readonly createWorker: () => Worker
private readonly getExecutionContext: () => Promise<ExecutionContext | null>
private readonly fileUploader: FileUploader
private listeners = new Set<Listener>()
private activeExecution: { id: string; startedAt: number } | null = null
private state: PythonRunnerState = DEFAULT_STATE
constructor(
fileId: string,
baseAssetPath: string,
getExecutionContext: () => Promise<ExecutionContext | null>,
createWorker: () => Worker,
fileUploader: FileUploader
) {
this.fileId = fileId
this.baseAssetPath = baseAssetPath
this.createWorker = createWorker
this.getExecutionContext = getExecutionContext
this.fileUploader = fileUploader
}
subscribe = (listener: Listener): (() => void) => {
this.listeners.add(listener)
return () => this.listeners.delete(listener)
}
getState = () => this.state
private updateState(fields: Partial<PythonRunnerState>) {
const prev = this.state
const output = fields.output ?? prev.output
const status = fields.status ?? prev.status
const error = fields.error !== undefined ? fields.error : prev.error
if (
output === prev.output &&
status === prev.status &&
error === prev.error
) {
return
}
this.state = { output, status, error }
for (const listener of this.listeners) {
listener()
}
}
init() {
if (this.client) {
return
}
this.updateState({ status: 'loading', error: null })
this.client = new PyodideWorkerClient({
baseAssetPath: this.baseAssetPath,
createWorker: this.createWorker,
fileUploader: this.fileUploader,
onLifecycle: event => {
switch (event.type) {
case 'loaded':
this.updateState({ status: 'idle', error: null })
return
case 'loading-failed':
debugConsole.error('Failed to load Python runtime', event.error)
this.updateState({ status: 'errored', error: event.error })
return
case 'run-finished': {
const active = this.activeExecution
if (
event.fileId !== this.fileId ||
active?.id !== event.executionId
) {
return
}
this.activeExecution = null
sendMB('script-runner-execution-completed', {
result: event.success ? 'success' : 'error',
errorType: event.success ? undefined : event.errorType,
executionTimeMs: Math.round(performance.now() - active.startedAt),
filesImportedCount: event.imports.length,
filesImportedExtensions: collectExtensions(event.imports),
filesWrittenCount: event.outputs.length,
filesWrittenExtensions: collectExtensions(event.outputs),
})
// event.outputs are full worker paths (/project/foo.txt) while
// event.failedUploads are relativePaths (foo.txt); strip the
// prefix before comparing.
const failed = new Set(event.failedUploads)
const uploadedPaths = event.outputs
.map(stripProjectFsPrefix)
.filter(p => !failed.has(p))
if (uploadedPaths.length > 0) {
showPythonFilesSavedToast(uploadedPaths)
}
this.updateState({ status: 'finished' })
}
}
},
onOutput: (stream, line, fileId, executionId) => {
if (
fileId !== this.fileId ||
this.activeExecution?.id !== executionId
) {
return
}
this.updateState({
output: appendCapped(this.state.output, { stream, line }),
})
},
})
}
async run() {
if (!this.client || this.state.status === 'loading') {
return
}
let context: ExecutionContext | null
try {
context = await this.getExecutionContext()
} catch (err) {
debugConsole.error('Failed to build execution context', err)
this.updateState({ status: 'errored', error: formatError(err) })
return
}
// Re-check after await — status may have changed but TypeScript
// still narrows from the pre-await check, so we cast back.
if (
!context ||
!this.client ||
(this.state.status as ExecutionStatus) === 'loading'
) {
return
}
const { code, files } = context
sendMB('script-runner-run-clicked', {
scriptLineCount: countLines(code),
})
const executionId = uuid()
this.activeExecution = { id: executionId, startedAt: performance.now() }
this.updateState({ status: 'running', output: [], error: null })
try {
this.client.runCode(code, {
fileId: this.fileId,
executionId,
files,
})
} catch (runError) {
if (this.activeExecution?.id !== executionId) {
return
}
this.activeExecution = null
this.updateState({ status: 'errored', error: formatError(runError) })
}
}
interrupt() {
if (!this.client) {
return
}
if (this.state.status === 'running' && this.activeExecution) {
sendMB('script-runner-stop-clicked', {
timeBeforeStopMs: Math.round(
performance.now() - this.activeExecution.startedAt
),
})
}
this.client.reset()
this.activeExecution = null
// The worker is terminated and recreated by reset(), so it needs to
// reload Pyodide. The 'loaded' lifecycle callback will transition
// back to 'idle'.
this.updateState({
status: 'loading',
output:
this.state.status === 'running'
? appendCapped(this.state.output, {
stream: 'info',
line: 'Execution interrupted',
})
: this.state.output,
})
}
destroy() {
if (this.client) {
this.client.destroy()
this.client = null
}
}
}
function appendCapped(existing: OutputLine[], entry: OutputLine): OutputLine[] {
const updated = [...existing, entry]
return updated.length > MAX_OUTPUT_LINES
? updated.slice(-MAX_OUTPUT_LINES)
: updated
}
function formatError(error: unknown): string {
if (error instanceof Error) {
return error.message
}
return String(error)
}
function countLines(code: string): number {
if (code.length === 0) {
return 0
}
return code.split('\n').length
}
function extractExtension(filePath: string): string {
return path.extname(filePath).slice(1).toLowerCase()
}
function collectExtensions(filePaths: string[]): string {
const seen = new Set<string>()
for (const filePath of filePaths) {
const ext = extractExtension(filePath)
if (ext) {
seen.add(ext)
}
}
return Array.from(seen).join(',')
}
services/web/frontend/js/features/ide-react/components/global-toasts.tsx
-2
View File
@@ -8,7 +8,6 @@ import { OLToastContainer } from '@/shared/components/ol/ol-toast-container'
import clipboardToastGenerators from '@/features/source-editor/components/clipboard-toasts' import clipboardToastGenerators from '@/features/source-editor/components/clipboard-toasts'
import importDocumentFeedbackToastGenerators from '@/features/project-list/components/new-project-button/import-document-feedback-toast' import importDocumentFeedbackToastGenerators from '@/features/project-list/components/new-project-button/import-document-feedback-toast'
import exportDocumentToastGenerators from '@/features/ide-react/components/toolbar/export-document-toasts' import exportDocumentToastGenerators from '@/features/ide-react/components/toolbar/export-document-toasts'
import pythonOutputToastGenerators from '@/features/ide-react/components/editor/python/python-output-toasts'
const moduleGeneratorsImport = importOverleafModules('toastGenerators') as { const moduleGeneratorsImport = importOverleafModules('toastGenerators') as {
import: { default: GlobalToastGeneratorEntry[] } import: { default: GlobalToastGeneratorEntry[] }
@@ -32,7 +31,6 @@ const GENERATOR_LIST: GlobalToastGeneratorEntry[] = [
...clipboardToastGenerators, ...clipboardToastGenerators,
...importDocumentFeedbackToastGenerators, ...importDocumentFeedbackToastGenerators,
...exportDocumentToastGenerators, ...exportDocumentToastGenerators,
...pythonOutputToastGenerators,
] ]
const GENERATOR_MAP: Map<string, GlobalToastGenerator> = new Map( const GENERATOR_MAP: Map<string, GlobalToastGenerator> = new Map(
GENERATOR_LIST.map(({ key, generator }) => [key, generator]) GENERATOR_LIST.map(({ key, generator }) => [key, generator])
services/web/frontend/js/features/ide-react/components/layout/editor.tsx
+14 -8
View File
@@ -5,22 +5,22 @@ import classNames from 'classnames'
import SourceEditor from '@/features/source-editor/components/source-editor' import SourceEditor from '@/features/source-editor/components/source-editor'
import { Panel, PanelGroup } from 'react-resizable-panels' import { Panel, PanelGroup } from 'react-resizable-panels'
import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle' import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle'
import { Suspense } from 'react' import { FC, Suspense } from 'react'
import { FullSizeLoadingSpinner } from '@/shared/components/loading-spinner' import { FullSizeLoadingSpinner } from '@/shared/components/loading-spinner'
import SymbolPalettePane from '@/features/ide-react/components/editor/symbol-palette-pane' import SymbolPalettePane from '@/features/ide-react/components/editor/symbol-palette-pane'
import { useEditorPropertiesContext } from '@/features/ide-react/context/editor-properties-context' import { useEditorPropertiesContext } from '@/features/ide-react/context/editor-properties-context'
import { PythonEditorSplit } from '@/features/ide-react/components/layout/python-editor-split'
import { isSplitTestEnabled } from '@/utils/splitTestUtils' import { isSplitTestEnabled } from '@/utils/splitTestUtils'
import importOverleafModules from '../../../../../macros/import-overleaf-module.macro'
const [pythonRunnerModule] = importOverleafModules('pythonRunner') as {
import: { PythonEditorSplit: FC }
}[]
export const Editor = () => { export const Editor = () => {
const { opening, errorState, showSymbolPalette } = const { opening, errorState, showSymbolPalette } =
useEditorPropertiesContext() useEditorPropertiesContext()
const { selectedEntityCount, openEntity } = useFileTreeOpenContext() const { selectedEntityCount, openEntity } = useFileTreeOpenContext()
const { currentDocumentId, currentDocument } = useEditorOpenDocContext() const { currentDocumentId, currentDocument } = useEditorOpenDocContext()
const isPythonDocument =
openEntity?.type === 'doc' &&
openEntity.entity.name.toLowerCase().endsWith('.py')
const pythonExecutionEnabled = isSplitTestEnabled('overleaf-code')
if (!currentDocumentId) { if (!currentDocumentId) {
return null return null
@@ -30,6 +30,10 @@ export const Editor = () => {
(!currentDocument || opening) && !errorState && currentDocumentId (!currentDocument || opening) && !errorState && currentDocumentId
) )
const isPythonDocument =
openEntity?.type === 'doc' &&
openEntity.entity.name.toLowerCase().endsWith('.py')
return ( return (
<div <div
className={classNames('ide-redesign-editor-content', { className={classNames('ide-redesign-editor-content', {
@@ -45,8 +49,10 @@ export const Editor = () => {
order={1} order={1}
className="ide-redesign-editor-panel" className="ide-redesign-editor-panel"
> >
{isPythonDocument && pythonExecutionEnabled ? ( {pythonRunnerModule &&
<PythonEditorSplit /> isPythonDocument &&
isSplitTestEnabled('overleaf-code') ? (
<pythonRunnerModule.import.PythonEditorSplit />
) : ( ) : (
<SourceEditor /> <SourceEditor />
)} )}
services/web/frontend/js/features/ide-react/components/layout/python-editor-split.tsx
-30
View File
@@ -1,30 +0,0 @@
import { Panel, PanelGroup } from 'react-resizable-panels'
import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle'
import PythonOutputPane from '@/features/ide-react/components/editor/python/python-output-pane'
import SourceEditor from '@/features/source-editor/components/source-editor'
import { PythonExecutionProvider } from '@/features/ide-react/context/python-execution-context'
export const PythonEditorSplit = () => {
return (
<PythonExecutionProvider>
<PanelGroup
autoSaveId="ide-redesign-editor-python-output"
direction="vertical"
className="ide-redesign-python-editor-split"
>
<Panel id="ide-redesign-panel-source-editor-content" order={1}>
<SourceEditor />
</Panel>
<VerticalResizeHandle id="ide-redesign-editor-python-output" />
<Panel
id="ide-redesign-panel-python-output"
order={2}
defaultSize={35}
minSize={10}
>
<PythonOutputPane />
</Panel>
</PanelGroup>
</PythonExecutionProvider>
)
}
services/web/frontend/js/features/ide-react/context/python-execution-context.tsx
-167
View File
@@ -1,167 +0,0 @@
import {
createContext,
FC,
PropsWithChildren,
useCallback,
useContext,
useEffect,
useMemo,
useRef,
} from 'react'
import getMeta from '@/utils/meta'
import { useFileTreePathContext } from '@/features/file-tree/contexts/file-tree-path'
import { useEditorManagerContext } from '@/features/ide-react/context/editor-manager-context'
import { useProjectContext } from '@/shared/context/project-context'
import { useFileTreeData } from '@/shared/context/file-tree-data-context'
import {
uploadBatch,
BatchUploadItem,
} from '@/infrastructure/batch-file-uploader'
import {
PythonRunner,
ExecutionContext,
} from '@/features/ide-react/components/editor/python/python-runner'
// Worker factory lives here (a .tsx file) so that the full
// `new Worker(new URL(..., import.meta.url))` expression is in a single place
// where webpack 5 can statically detect it and create a proper worker bundle.
// Keeping import.meta.url out of .ts files also avoids Node.js 24 switching to
// ESM mode and breaking CJS-based test loading via @babel/register.
const createPyodideWorker = () =>
new Worker(
/* webpackChunkName: "pyodide-worker" */
new URL('../components/editor/python/pyodide.worker.ts', import.meta.url),
{ type: 'module' }
)
export interface PythonExecutionContextValue {
getPythonRunner: (fileId: string) => PythonRunner
}
export const PythonExecutionContext = createContext<
PythonExecutionContextValue | undefined
>(undefined)
export const PythonExecutionProvider: FC<PropsWithChildren> = ({
children,
}) => {
const { openDocs } = useEditorManagerContext()
const { projectId, projectSnapshot } = useProjectContext()
const { pathInFolder } = useFileTreePathContext()
const { fileTreeData } = useFileTreeData()
const runnersRef = useRef(new Map<string, PythonRunner>())
const baseAssetPathRef = useRef<string | null>(null)
const pathInFolderRef = useRef(pathInFolder)
pathInFolderRef.current = pathInFolder
// Ref so the upload closure built into each PythonRunner reads the
// current value at call time rather than capturing a potentially-stale
// value from when the runner was constructed (fileTreeData may load
// after the runner is created).
const fileTreeDataRef = useRef(fileTreeData)
fileTreeDataRef.current = fileTreeData
// Refreshes the project snapshot and resolves the source code and all project
// files for the given fileId, to be passed to the executor for running.
const getExecutionContext = useCallback(
async (fileId: string): Promise<ExecutionContext | null> => {
await openDocs.awaitBufferedOps(AbortSignal.timeout(5000))
await projectSnapshot.refresh()
const relativePath = pathInFolderRef.current(fileId)
if (!relativePath) {
return null
}
const code = projectSnapshot.getDocContents(relativePath)
if (code == null) {
return null
}
const docPaths = projectSnapshot.getDocPaths()
const files = docPaths
.map(docPath => {
const content = projectSnapshot.getDocContents(docPath)
return content != null ? { relativePath: docPath, content } : null
})
.filter(
(f): f is { relativePath: string; content: string } => f != null
)
return { code, files }
},
[openDocs, projectSnapshot]
)
const getPythonRunner = useCallback(
(fileId: string): PythonRunner => {
const existing = runnersRef.current.get(fileId)
if (existing) {
return existing
}
if (!baseAssetPathRef.current) {
baseAssetPathRef.current = new URL(
getMeta('ol-baseAssetPath'),
window.location.href
).toString()
}
const uploadOutputFiles = (items: BatchUploadItem[]) => {
const folderId = fileTreeDataRef.current?._id
if (!folderId) {
return Promise.reject(
new Error('File tree not loaded; cannot upload output files')
)
}
return uploadBatch(items, {
projectId,
folderId,
})
}
const runner = new PythonRunner(
fileId,
baseAssetPathRef.current,
() => getExecutionContext(fileId),
createPyodideWorker,
uploadOutputFiles
)
runner.init()
runnersRef.current.set(fileId, runner)
return runner
},
[getExecutionContext, projectId]
)
useEffect(() => {
const runners = runnersRef.current
return () => {
for (const runner of runners.values()) {
runner.destroy()
}
runners.clear()
}
}, [])
const value = useMemo(() => ({ getPythonRunner }), [getPythonRunner])
return (
<PythonExecutionContext.Provider value={value}>
{children}
</PythonExecutionContext.Provider>
)
}
export const usePythonExecutionContext = (): PythonExecutionContextValue => {
const context = useContext(PythonExecutionContext)
if (!context) {
throw new Error(
'usePythonExecutionContext is only available inside PythonExecutionContext.Provider'
)
}
return context
}
services/web/frontend/stylesheets/pages/editor/ide-redesign.scss
-79
View File
@@ -57,85 +57,6 @@
position: relative; position: relative;
} }
.ide-redesign-python-editor-split {
height: 100%;
}
.ide-redesign-python-output-pane {
height: 100%;
display: flex;
flex-direction: column;
background-color: var(--ide-redesign-background);
color: var(--ide-redesign-color);
border-top: 1px solid var(--border-divider);
}
.ide-redesign-python-output-pane-toolbar {
display: flex;
align-items: center;
height: var(--toolbar-height);
border-bottom: 1px solid var(--toolbar-border-color);
}
.ide-redesign-python-output-pane-toolbar-left {
display: flex;
align-items: center;
align-self: stretch;
gap: var(--spacing-02);
}
.ide-redesign-python-output-pane-run-button-wrapper {
height: 24px;
background-color: var(--bg-accent-01);
border-radius: var(--ds-border-radius-300);
margin-left: var(--spacing-02);
&.compile-button-group-running {
background-color: var(--bg-danger-01);
}
.btn-primary:hover {
z-index: auto;
}
}
.ide-redesign-python-output-pane-body {
flex: 1;
overflow: auto;
padding: var(--spacing-06);
font-family: 'DM Mono', monospace;
font-size: var(--font-size-02);
line-height: var(--line-height-03);
}
.ide-redesign-python-output-pane-placeholder {
color: var(--content-secondary);
}
.ide-redesign-python-output-pane-line {
white-space: pre-wrap;
word-break: break-word;
min-height: var(--line-height-03);
}
.ide-redesign-python-output-pane-line-info {
color: var(--content-info);
}
.ide-redesign-python-output-pane-line-stderr {
color: var(--red-50);
}
.ide-redesign-python-output-pane-placeholder,
.ide-redesign-python-output-pane-error {
white-space: pre-wrap;
}
.ide-redesign-python-output-pane-error {
margin-bottom: var(--spacing-03);
color: var(--red-50);
}
.ide-redesign-labs-user-beta-promo { .ide-redesign-labs-user-beta-promo {
position: absolute; position: absolute;
top: 60px; top: 60px;
services/web/test/frontend/features/ide-react/components/python-output-pane.spec.tsx
-481
View File
@@ -1,481 +0,0 @@
import React, { FC, PropsWithChildren } from 'react'
import PythonOutputPane from '@/features/ide-react/components/editor/python/python-output-pane'
import {
EditorProviders,
projectDefaults,
} from '../../../helpers/editor-providers'
import { FileTreePathContext } from '@/features/file-tree/contexts/file-tree-path'
import { ProjectContext } from '@/shared/context/project-context'
import { ProjectSnapshot } from '@/infrastructure/project-snapshot'
import { PythonExecutionProvider } from '@/features/ide-react/context/python-execution-context'
const pythonExecutableScript: Record<string, string> = {
file_id: 'test-py-doc-id',
filename: 'test.py',
}
const FileTreePathProvider: FC<PropsWithChildren> = ({ children }) => {
return (
<FileTreePathContext.Provider
value={{
pathInFolder: () => pythonExecutableScript.filename,
findEntityByPath: () => null,
previewByPath: () => null,
dirname: () => null,
}}
>
{children}
</FileTreePathContext.Provider>
)
}
function makeProjectProvider(fileContents: Record<string, string>) {
const ProjectProvider: FC<PropsWithChildren> = ({ children }) => {
const projectSnapshot = {
refresh: async () => {},
getDocPaths: () => Object.keys(fileContents),
getDocContents: (path: string) => fileContents[path] ?? null,
} as unknown as ProjectSnapshot
return (
<ProjectContext.Provider
value={{
projectId: projectDefaults._id,
project: projectDefaults,
joinProject: () => {},
updateProject: () => {},
joinedOnce: true,
projectSnapshot,
tags: [],
features: projectDefaults.features,
name: projectDefaults.name,
}}
>
{children}
</ProjectContext.Provider>
)
}
return ProjectProvider
}
describe('<PythonOutputPane />', function () {
beforeEach(function () {
window.metaAttributesCache.set('ol-baseAssetPath', '/__cypress/src/')
})
it('executes a Python script and displays its output', function () {
const executablePythonFileContents = "print('hello!')"
const projectFiles = {
[pythonExecutableScript.filename]: executablePythonFileContents,
}
const ProjectProvider = makeProjectProvider(projectFiles)
cy.mount(
<EditorProviders
scope={{
editor: {
sharejs_doc: {
doc_id: pythonExecutableScript.file_id,
getSnapshot: () => executablePythonFileContents,
},
currentDocumentId: pythonExecutableScript.file_id,
openDocName: pythonExecutableScript.filename,
},
}}
providers={{ FileTreePathProvider, ProjectProvider }}
>
<PythonExecutionProvider>
<PythonOutputPane />
</PythonExecutionProvider>
</EditorProviders>
)
cy.findByRole('button', { name: 'Run Python code' })
.should('not.be.disabled')
.click()
cy.findByText('hello!').should('exist')
})
it('can import and use values from other project Python files', function () {
const executablePythonFileContents =
'from message import message\nprint(message)'
const importedPythonFile = {
filename: 'message.py',
file_contents: "message = 'hello!'",
}
const projectFiles = {
[pythonExecutableScript.filename]: executablePythonFileContents,
[importedPythonFile.filename]: importedPythonFile.file_contents,
}
const ProjectProvider = makeProjectProvider(projectFiles)
cy.mount(
<EditorProviders
scope={{
editor: {
sharejs_doc: {
doc_id: pythonExecutableScript.file_id,
getSnapshot: () => executablePythonFileContents,
},
currentDocumentId: pythonExecutableScript.file_id,
openDocName: pythonExecutableScript.filename,
},
}}
providers={{ FileTreePathProvider, ProjectProvider }}
>
<PythonExecutionProvider>
<PythonOutputPane />
</PythonExecutionProvider>
</EditorProviders>
)
cy.findByRole('button', { name: 'Run Python code' })
.should('not.be.disabled')
.click()
cy.findByText('hello!').should('exist')
})
it('can import files from different directories relative to the executable script', function () {
const executablePythonFileContents = [
'from scripts.data_importers.csv_importer import print_data',
'print_data()',
].join('\n')
const csvImporterFile = {
filename: 'scripts/data_importers/csv_importer.py',
file_contents: [
'import csv',
'',
'def print_data():',
' with open("food_items.csv", "r") as f:',
' reader = csv.reader(f)',
' for row in reader:',
' print(",".join(row))',
].join('\n'),
}
const csvDataFile = {
filename: 'food_items.csv',
file_contents: 'name,type\nPizza,Italian\nSushi,Japanese\nTacos,Mexican',
}
const projectFiles = {
'scripts/test.py': executablePythonFileContents,
[csvImporterFile.filename]: csvImporterFile.file_contents,
[csvDataFile.filename]: csvDataFile.file_contents,
}
const ProjectProvider = makeProjectProvider(projectFiles)
const NestedFileTreePathProvider: FC<PropsWithChildren> = ({
children,
}) => (
<FileTreePathContext.Provider
value={{
pathInFolder: () => 'scripts/test.py',
findEntityByPath: () => null,
previewByPath: () => null,
dirname: () => null,
}}
>
{children}
</FileTreePathContext.Provider>
)
cy.mount(
<EditorProviders
scope={{
editor: {
sharejs_doc: {
doc_id: pythonExecutableScript.file_id,
getSnapshot: () => executablePythonFileContents,
},
currentDocumentId: pythonExecutableScript.file_id,
openDocName: 'test.py',
},
}}
providers={{
FileTreePathProvider: NestedFileTreePathProvider,
ProjectProvider,
}}
>
<PythonExecutionProvider>
<PythonOutputPane />
</PythonExecutionProvider>
</EditorProviders>
)
cy.findByRole('button', { name: 'Run Python code' })
.should('not.be.disabled')
.click()
cy.findByText('name,type').should('exist')
cy.findByText('Pizza,Italian').should('exist')
cy.findByText('Sushi,Japanese').should('exist')
cy.findByText('Tacos,Mexican').should('exist')
})
it('renders stderr output with the stderr line class', function () {
const executablePythonFileContents = [
'import sys',
"print('hello!')",
"sys.stderr.write('boom\\n')",
].join('\n')
const projectFiles = {
[pythonExecutableScript.filename]: executablePythonFileContents,
}
const ProjectProvider = makeProjectProvider(projectFiles)
cy.mount(
<EditorProviders
scope={{
editor: {
sharejs_doc: {
doc_id: pythonExecutableScript.file_id,
getSnapshot: () => executablePythonFileContents,
},
currentDocumentId: pythonExecutableScript.file_id,
openDocName: pythonExecutableScript.filename,
},
}}
providers={{ FileTreePathProvider, ProjectProvider }}
>
<PythonExecutionProvider>
<PythonOutputPane />
</PythonExecutionProvider>
</EditorProviders>
)
cy.findByRole('button', { name: 'Run Python code' })
.should('not.be.disabled')
.click()
cy.findByText('hello!')
.should('have.class', 'ide-redesign-python-output-pane-line-stdout')
.and('not.have.class', 'ide-redesign-python-output-pane-line-stderr')
cy.findByText('boom').should(
'have.class',
'ide-redesign-python-output-pane-line-stderr'
)
cy.findByText("Only Pyodide's built-in packages", { exact: false }).should(
'not.exist'
)
})
it('renders the interrupt message as an info line', function () {
const executablePythonFileContents = 'while True:\n pass\n'
const projectFiles = {
[pythonExecutableScript.filename]: executablePythonFileContents,
}
const ProjectProvider = makeProjectProvider(projectFiles)
cy.mount(
<EditorProviders
scope={{
editor: {
sharejs_doc: {
doc_id: pythonExecutableScript.file_id,
getSnapshot: () => executablePythonFileContents,
},
currentDocumentId: pythonExecutableScript.file_id,
openDocName: pythonExecutableScript.filename,
},
}}
providers={{ FileTreePathProvider, ProjectProvider }}
>
<PythonExecutionProvider>
<PythonOutputPane />
</PythonExecutionProvider>
</EditorProviders>
)
cy.findByRole('button', { name: 'Run Python code' })
.should('not.be.disabled')
.click()
cy.findByRole('button', { name: 'Stop Python execution' })
.should('not.be.disabled')
.click()
cy.findByText('Execution interrupted').should(
'have.class',
'ide-redesign-python-output-pane-line-info'
)
})
it('renders stdout, stderr, and info lines with visually distinct CSS classes', function () {
const executablePythonFileContents = [
'import sys',
"print('hello!')",
"sys.stderr.write('boom\\n')",
'while True:',
' pass',
].join('\n')
const projectFiles = {
[pythonExecutableScript.filename]: executablePythonFileContents,
}
const ProjectProvider = makeProjectProvider(projectFiles)
cy.mount(
<EditorProviders
scope={{
editor: {
sharejs_doc: {
doc_id: pythonExecutableScript.file_id,
getSnapshot: () => executablePythonFileContents,
},
currentDocumentId: pythonExecutableScript.file_id,
openDocName: pythonExecutableScript.filename,
},
}}
providers={{ FileTreePathProvider, ProjectProvider }}
>
<PythonExecutionProvider>
<PythonOutputPane />
</PythonExecutionProvider>
</EditorProviders>
)
cy.findByRole('button', { name: 'Run Python code' })
.should('not.be.disabled')
.click()
cy.findByText('hello!')
.should('have.class', 'ide-redesign-python-output-pane-line-stdout')
.and('not.have.class', 'ide-redesign-python-output-pane-line-stderr')
.and('not.have.class', 'ide-redesign-python-output-pane-line-info')
cy.findByText('boom')
.should('have.class', 'ide-redesign-python-output-pane-line-stderr')
.and('not.have.class', 'ide-redesign-python-output-pane-line-stdout')
.and('not.have.class', 'ide-redesign-python-output-pane-line-info')
cy.findByRole('button', { name: 'Stop Python execution' })
.should('not.be.disabled')
.click()
cy.findByText('Execution interrupted')
.should('have.class', 'ide-redesign-python-output-pane-line-info')
.and('not.have.class', 'ide-redesign-python-output-pane-line-stdout')
.and('not.have.class', 'ide-redesign-python-output-pane-line-stderr')
})
it('can load common python data analysis packages on code execution', function () {
const executablePythonFileContents = [
'import tomli',
'',
"print(tomli.loads('greeting = \"hello from tomli\"')['greeting'])",
].join('\n')
const projectFiles = {
[pythonExecutableScript.filename]: executablePythonFileContents,
}
const ProjectProvider = makeProjectProvider(projectFiles)
cy.mount(
<EditorProviders
scope={{
editor: {
sharejs_doc: {
doc_id: pythonExecutableScript.file_id,
getSnapshot: () => executablePythonFileContents,
},
currentDocumentId: pythonExecutableScript.file_id,
openDocName: pythonExecutableScript.filename,
},
}}
providers={{ FileTreePathProvider, ProjectProvider }}
>
<PythonExecutionProvider>
<PythonOutputPane />
</PythonExecutionProvider>
</EditorProviders>
)
cy.findByRole('button', { name: 'Run Python code' })
.should('not.be.disabled')
.click()
cy.findByText("ModuleNotFoundError: No module named 'tomli'").should(
'not.exist'
)
cy.findByText('hello from tomli').should('exist')
})
it('auto-installs python packages imported by the executing script', function () {
const executablePythonFileContents = [
'import tomli',
'',
"print(tomli.loads('greeting = \"hello from tomli\"')['greeting'])",
].join('\n')
const projectFiles = {
[pythonExecutableScript.filename]: executablePythonFileContents,
}
const ProjectProvider = makeProjectProvider(projectFiles)
cy.mount(
<EditorProviders
scope={{
editor: {
sharejs_doc: {
doc_id: pythonExecutableScript.file_id,
getSnapshot: () => executablePythonFileContents,
},
currentDocumentId: pythonExecutableScript.file_id,
openDocName: pythonExecutableScript.filename,
},
}}
providers={{ FileTreePathProvider, ProjectProvider }}
>
<PythonExecutionProvider>
<PythonOutputPane />
</PythonExecutionProvider>
</EditorProviders>
)
cy.findByRole('button', { name: 'Run Python code' })
.should('not.be.disabled')
.click()
cy.findByText("ModuleNotFoundError: No module named 'tomli'").should(
'not.exist'
)
cy.findByText('hello from tomli').should('exist')
})
it('augments ModuleNotFoundError output with help text about supported Pyodide packages', function () {
const executablePythonFileContents =
'import this_module_does_not_exist_in_pyodide\n'
const projectFiles = {
[pythonExecutableScript.filename]: executablePythonFileContents,
}
const ProjectProvider = makeProjectProvider(projectFiles)
cy.mount(
<EditorProviders
scope={{
editor: {
sharejs_doc: {
doc_id: pythonExecutableScript.file_id,
getSnapshot: () => executablePythonFileContents,
},
currentDocumentId: pythonExecutableScript.file_id,
openDocName: pythonExecutableScript.filename,
},
}}
providers={{ FileTreePathProvider, ProjectProvider }}
>
<PythonExecutionProvider>
<PythonOutputPane />
</PythonExecutionProvider>
</EditorProviders>
)
cy.findByRole('button', { name: 'Run Python code' })
.should('not.be.disabled')
.click()
cy.findByText('ModuleNotFoundError', { exact: false }).should('exist')
cy.findByText("Only Pyodide's built-in packages", { exact: false }).should(
'have.class',
'ide-redesign-python-output-pane-line-stderr'
)
})
})
services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts
-778
View File
@@ -1,778 +0,0 @@
import { expect } from 'chai'
import sinon from 'sinon'
import {
PyodideWorkerClient,
type LifecycleCallback,
type OutputCallback,
} from '@/features/ide-react/components/editor/python/pyodide-worker-client'
import type { FileUploader } from '@/features/ide-react/components/editor/python/python-runner'
import { WorkerMock, createWorker } from './worker-mock'
const BASE_ASSET_PATH = 'https://assets.example.test/'
const fileUploaderStub: FileUploader = () => Promise.resolve([])
describe('PyodideWorkerClient', function () {
beforeEach(function () {
WorkerMock.instances.length = 0
})
it('queues runCode until the worker reports listening', function () {
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
fileUploader: fileUploaderStub,
})
const worker = WorkerMock.instances[0]
client.runCode('print("ok")', {
fileId: 'main.py',
executionId: 'exec-1',
files: [{ relativePath: 'main.py', content: 'print("ok")' }],
})
expect(worker.postedMessages).to.have.length(0)
worker.emitMessage({ type: 'listening' })
expect(worker.postedMessages.map(message => message.type)).to.deep.equal([
'init',
'run-code',
])
const runRequest = worker.postedMessages.find(
message => message.type === 'run-code'
)
expect(runRequest).to.include({
type: 'run-code',
fileId: 'main.py',
executionId: 'exec-1',
code: 'print("ok")',
})
expect(runRequest.files).to.deep.equal([
{ relativePath: 'main.py', content: 'print("ok")' },
])
})
it('sends runCode as fire-and-forget', function () {
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
fileUploader: fileUploaderStub,
})
const worker = WorkerMock.instances[0]
worker.emitMessage({ type: 'listening' })
client.runCode('raise RuntimeError("boom")', {
fileId: 'boom.py',
executionId: 'exec-2',
files: [],
})
const runRequest = worker.postedMessages.find(
message => message.type === 'run-code'
)
expect(runRequest).to.include({
type: 'run-code',
fileId: 'boom.py',
executionId: 'exec-2',
})
})
function setupClientWithLifecycleTracking() {
const lifecycleEvents: Parameters<LifecycleCallback>[0][] = []
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
onLifecycle: event => {
lifecycleEvents.push(event)
},
fileUploader: fileUploaderStub,
})
const worker = WorkerMock.instances[0]
worker.emitMessage({ type: 'listening' })
return { client, worker, lifecycleEvents }
}
function setupClientWithUploadTracking(options: {
fileUploader: FileUploader
}) {
const lifecycleEvents: Parameters<LifecycleCallback>[0][] = []
const outputCalls: Parameters<OutputCallback>[] = []
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
onLifecycle: event => {
lifecycleEvents.push(event)
},
onOutput: (...args) => {
outputCalls.push(args)
},
fileUploader: options.fileUploader,
})
const worker = WorkerMock.instances[0]
worker.emitMessage({ type: 'listening' })
return { client, worker, lifecycleEvents, outputCalls }
}
async function waitFor(predicate: () => boolean, timeoutMs = 200) {
const deadline = Date.now() + timeoutMs
while (!predicate()) {
if (Date.now() > deadline) {
throw new Error('waitFor timed out')
}
await new Promise(resolve => setTimeout(resolve, 0))
}
}
it('emits run-finished lifecycle event from run-code-result', function () {
const { client, worker, lifecycleEvents } =
setupClientWithLifecycleTracking()
client.runCode('print("ok")', {
fileId: 'main.py',
executionId: 'exec-3',
files: [],
})
worker.emitMessage({
type: 'run-code-result',
fileId: 'main.py',
executionId: 'exec-3',
success: true,
outputs: ['/project/output.txt'],
outputFiles: [],
imports: [],
})
expect(lifecycleEvents).to.deep.equal([
{
type: 'run-finished',
fileId: 'main.py',
executionId: 'exec-3',
success: true,
outputs: ['/project/output.txt'],
failedUploads: [],
imports: [],
errorType: undefined,
},
])
})
it('surfaces outputs array from run-code-result with multiple files', function () {
const { client, worker, lifecycleEvents } =
setupClientWithLifecycleTracking()
client.runCode('write_files()', {
fileId: 'main.py',
executionId: 'exec-4',
files: [],
})
worker.emitMessage({
type: 'run-code-result',
fileId: 'main.py',
executionId: 'exec-4',
success: true,
outputs: ['/project/fig1.png', '/project/results/data.csv'],
outputFiles: [],
imports: [],
})
expect(lifecycleEvents).to.deep.equal([
{
type: 'run-finished',
fileId: 'main.py',
executionId: 'exec-4',
success: true,
outputs: ['/project/fig1.png', '/project/results/data.csv'],
failedUploads: [],
imports: [],
errorType: undefined,
},
])
})
it('surfaces empty outputs when no project files were written', function () {
const { client, worker, lifecycleEvents } =
setupClientWithLifecycleTracking()
client.runCode('print("no writes")', {
fileId: 'main.py',
executionId: 'exec-5',
files: [],
})
worker.emitMessage({
type: 'run-code-result',
fileId: 'main.py',
executionId: 'exec-5',
success: true,
outputs: [],
outputFiles: [],
imports: [],
})
expect(lifecycleEvents).to.deep.equal([
{
type: 'run-finished',
fileId: 'main.py',
executionId: 'exec-5',
success: true,
outputs: [],
failedUploads: [],
imports: [],
errorType: undefined,
},
])
})
it('surfaces success and outputFiles from run-code-result', async function () {
const { client, worker, lifecycleEvents } =
setupClientWithLifecycleTracking()
client.runCode('write_files()', {
fileId: 'main.py',
executionId: 'exec-success',
files: [],
})
const csvContent = new Uint8Array([1, 2, 3])
const pngContent = new Uint8Array([4, 5, 6, 7])
worker.emitMessage({
type: 'run-code-result',
fileId: 'main.py',
executionId: 'exec-success',
success: true,
outputs: ['/project/data.csv', '/project/plot.png'],
outputFiles: [
{ relativePath: 'data.csv', content: csvContent },
{ relativePath: 'plot.png', content: pngContent },
],
imports: [],
})
await waitFor(() =>
Boolean(lifecycleEvents.find(e => e.type === 'run-finished'))
)
const finished = lifecycleEvents.find(e => e.type === 'run-finished')
expect(finished).to.deep.equal({
type: 'run-finished',
fileId: 'main.py',
executionId: 'exec-success',
success: true,
outputs: ['/project/data.csv', '/project/plot.png'],
failedUploads: [],
imports: [],
errorType: undefined,
})
})
it('propagates ModuleNotFoundError errorType on run-finished', function () {
const { client, worker, lifecycleEvents } =
setupClientWithLifecycleTracking()
client.runCode('import nope', {
fileId: 'main.py',
executionId: 'exec-mnfe',
files: [],
})
worker.emitMessage({
type: 'run-code-result',
fileId: 'main.py',
executionId: 'exec-mnfe',
success: false,
outputs: [],
outputFiles: [],
imports: [],
errorType: 'ModuleNotFoundError',
})
const finished = lifecycleEvents.find(e => e.type === 'run-finished')
expect(finished).to.deep.equal({
type: 'run-finished',
fileId: 'main.py',
executionId: 'exec-mnfe',
success: false,
outputs: [],
failedUploads: [],
imports: [],
errorType: 'ModuleNotFoundError',
})
})
it('surfaces success: false with empty outputFiles on script error', function () {
const { client, worker, lifecycleEvents } =
setupClientWithLifecycleTracking()
client.runCode('raise RuntimeError("boom")', {
fileId: 'main.py',
executionId: 'exec-error',
files: [],
})
worker.emitMessage({
type: 'run-code-result',
fileId: 'main.py',
executionId: 'exec-error',
success: false,
outputs: [],
outputFiles: [],
imports: [],
errorType: 'generic',
})
const finished = lifecycleEvents.find(e => e.type === 'run-finished')
expect(finished).to.deep.equal({
type: 'run-finished',
fileId: 'main.py',
executionId: 'exec-error',
success: false,
outputs: [],
failedUploads: [],
imports: [],
errorType: 'generic',
})
})
it('surfaces empty outputFiles when success but no files were written', function () {
const { client, worker, lifecycleEvents } =
setupClientWithLifecycleTracking()
client.runCode('print("no writes")', {
fileId: 'main.py',
executionId: 'exec-nowrites',
files: [],
})
worker.emitMessage({
type: 'run-code-result',
fileId: 'main.py',
executionId: 'exec-nowrites',
success: true,
outputs: [],
outputFiles: [],
imports: [],
})
const finished = lifecycleEvents.find(e => e.type === 'run-finished')
expect(finished).to.deep.equal({
type: 'run-finished',
fileId: 'main.py',
executionId: 'exec-nowrites',
success: true,
outputs: [],
failedUploads: [],
imports: [],
errorType: undefined,
})
})
it('reports lifecycle failure and rejects future run requests when loading fails', function () {
const lifecycleEvents: { type: string; error?: string }[] = []
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
onLifecycle: event => {
lifecycleEvents.push(event)
},
fileUploader: fileUploaderStub,
})
const worker = WorkerMock.instances[0]
worker.emitMessage({
type: 'loading-failed',
error: 'runtime unavailable',
})
expect(lifecycleEvents).to.deep.equal([
{ type: 'loading-failed', error: 'runtime unavailable' },
])
expect(() =>
client.runCode('print("ok")', {
fileId: 'main.py',
executionId: 'exec-4',
files: [],
})
).to.throw('runtime unavailable')
})
it('terminates the worker even when destroy is called after loading failure', function () {
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
fileUploader: fileUploaderStub,
})
const worker = WorkerMock.instances[0]
worker.emitMessage({
type: 'loading-failed',
error: 'runtime unavailable',
})
client.destroy()
expect(worker.terminated).to.equal(true)
})
context('upload behavior', function () {
const successResult = (name: string, relativePath: string) => ({
status: 'success' as const,
name,
relativePath,
data: { success: true },
})
const errorResult = (
name: string,
relativePath: string,
error: string
) => ({
status: 'error' as const,
name,
relativePath,
error,
})
function emitRunResult(
worker: WorkerMock,
executionId: string,
outputFiles: Array<{ relativePath: string; content: Uint8Array }>,
success = true
) {
worker.emitMessage({
type: 'run-code-result',
fileId: 'main.py',
executionId,
success,
outputs: [],
outputFiles,
imports: [],
})
}
const findFinished = (
lifecycleEvents: Parameters<LifecycleCallback>[0][]
) => lifecycleEvents.find(e => e.type === 'run-finished')
it('invokes fileUploader with mapped items when run-code-result has output files', async function () {
const uploader = sinon
.stub()
.resolves([successResult('data.csv', 'output/data.csv')])
const { worker, lifecycleEvents } = setupClientWithUploadTracking({
fileUploader: uploader,
})
emitRunResult(worker, 'exec-up-1', [
{
relativePath: 'output/data.csv',
content: new TextEncoder().encode('a,b\n1,2'),
},
])
await waitFor(() => Boolean(findFinished(lifecycleEvents)))
expect(uploader.calledOnce).to.be.true
const [items] = uploader.firstCall.args
expect(items).to.have.lengthOf(1)
expect(items[0].name).to.equal('data.csv')
expect(items[0].relativePath).to.equal('output/data.csv')
expect(items[0].file).to.be.instanceOf(Blob)
})
it('emits success: true and empty failedUploads when all uploads succeed', async function () {
const uploader = sinon
.stub()
.resolves([
successResult('a.csv', 'a.csv'),
successResult('b.csv', 'b.csv'),
])
const { worker, lifecycleEvents } = setupClientWithUploadTracking({
fileUploader: uploader,
})
emitRunResult(worker, 'exec-up-2', [
{ relativePath: 'a.csv', content: new TextEncoder().encode('1') },
{ relativePath: 'b.csv', content: new TextEncoder().encode('2') },
])
await waitFor(() => Boolean(findFinished(lifecycleEvents)))
const finished = findFinished(lifecycleEvents)
expect(finished).to.deep.include({ success: true, failedUploads: [] })
})
it('flips success to false and lists failed paths when an upload fails', async function () {
const uploader = sinon
.stub()
.resolves([
successResult('good.csv', 'good.csv'),
errorResult('bad.csv', 'output/bad.csv', 'duplicate_file_name'),
])
const { worker, lifecycleEvents, outputCalls } =
setupClientWithUploadTracking({ fileUploader: uploader })
emitRunResult(worker, 'exec-up-3', [
{ relativePath: 'good.csv', content: new TextEncoder().encode('1') },
{
relativePath: 'output/bad.csv',
content: new TextEncoder().encode('2'),
},
])
await waitFor(() => Boolean(findFinished(lifecycleEvents)))
const finished = findFinished(lifecycleEvents)
expect(finished).to.deep.include({
success: false,
failedUploads: ['output/bad.csv'],
errorType: 'UploadFileError',
})
// user-facing stderr line surfaced via outputCallback
expect(outputCalls).to.have.lengthOf(1)
const [stream, line] = outputCalls[0]
expect(stream).to.equal('stderr')
expect(line).to.include('output/bad.csv')
expect(line).to.include('duplicate_file_name')
})
it('lists every file in failedUploads and surfaces a single stderr line when fileUploader rejects', async function () {
const uploader = sinon.stub().rejects(new Error('network down'))
const { worker, lifecycleEvents, outputCalls } =
setupClientWithUploadTracking({ fileUploader: uploader })
emitRunResult(worker, 'exec-up-4', [
{ relativePath: 'a.csv', content: new TextEncoder().encode('1') },
{ relativePath: 'b.csv', content: new TextEncoder().encode('2') },
])
await waitFor(() => Boolean(findFinished(lifecycleEvents)))
const finished = findFinished(lifecycleEvents)
expect(finished).to.deep.include({
success: false,
errorType: 'UploadFileError',
})
expect(finished)
.to.have.property('failedUploads')
.that.has.members(['a.csv', 'b.csv'])
expect(outputCalls).to.have.lengthOf(1)
const [stream, line] = outputCalls[0]
expect(stream).to.equal('stderr')
expect(line).to.include('network down')
})
it('does not invoke fileUploader when run-code-result has success: false', async function () {
const uploader = sinon.stub().resolves([])
const { worker, lifecycleEvents } = setupClientWithUploadTracking({
fileUploader: uploader,
})
emitRunResult(
worker,
'exec-up-5',
[{ relativePath: 'a.csv', content: new TextEncoder().encode('1') }],
false
)
await waitFor(() => Boolean(findFinished(lifecycleEvents)))
expect(uploader.called).to.be.false
expect(findFinished(lifecycleEvents)).to.deep.include({
success: false,
failedUploads: [],
})
})
it('does not invoke fileUploader when outputFiles is empty', async function () {
const uploader = sinon.stub().resolves([])
const { worker, lifecycleEvents } = setupClientWithUploadTracking({
fileUploader: uploader,
})
emitRunResult(worker, 'exec-up-6', [])
await waitFor(() => Boolean(findFinished(lifecycleEvents)))
expect(uploader.called).to.be.false
})
})
describe('output-line forwarding', function () {
function setupClientWithOutputTracking() {
const outputCalls: Parameters<OutputCallback>[] = []
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
onOutput: (...args) => {
outputCalls.push(args)
},
fileUploader: fileUploaderStub,
})
const worker = WorkerMock.instances[0]
worker.emitMessage({ type: 'listening' })
return { client, worker, outputCalls }
}
it('forwards stdout output-line events to the output callback', function () {
const { worker, outputCalls } = setupClientWithOutputTracking()
worker.emitMessage({
type: 'output-line',
stream: 'stdout',
line: 'hello!',
fileId: 'main.py',
executionId: 'exec-stdout',
})
expect(outputCalls).to.deep.equal([
['stdout', 'hello!', 'main.py', 'exec-stdout'],
])
})
it('forwards stderr output-line events to the output callback', function () {
const { worker, outputCalls } = setupClientWithOutputTracking()
worker.emitMessage({
type: 'output-line',
stream: 'stderr',
line: 'boom',
fileId: 'main.py',
executionId: 'exec-stderr',
})
expect(outputCalls).to.deep.equal([
['stderr', 'boom', 'main.py', 'exec-stderr'],
])
})
it('forwards info output-line events to the output callback', function () {
const { worker, outputCalls } = setupClientWithOutputTracking()
worker.emitMessage({
type: 'output-line',
stream: 'info',
line: 'Loading numpy from package index',
fileId: 'main.py',
executionId: 'exec-info',
})
expect(outputCalls).to.deep.equal([
['info', 'Loading numpy from package index', 'main.py', 'exec-info'],
])
})
it('preserves stream type when forwarding stdout, stderr, and info in sequence', function () {
const { worker, outputCalls } = setupClientWithOutputTracking()
worker.emitMessage({
type: 'output-line',
stream: 'info',
line: 'Loading package',
fileId: 'main.py',
executionId: 'exec-mixed',
})
worker.emitMessage({
type: 'output-line',
stream: 'stdout',
line: 'result',
fileId: 'main.py',
executionId: 'exec-mixed',
})
worker.emitMessage({
type: 'output-line',
stream: 'stderr',
line: 'warning',
fileId: 'main.py',
executionId: 'exec-mixed',
})
expect(outputCalls.map(call => call[0])).to.deep.equal([
'info',
'stdout',
'stderr',
])
})
})
describe('reset', function () {
it('terminates the current worker and creates a new one', function () {
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
fileUploader: fileUploaderStub,
})
const originalWorker = WorkerMock.instances[0]
originalWorker.emitMessage({ type: 'listening' })
originalWorker.emitMessage({ type: 'loaded' })
client.reset()
expect(originalWorker.terminated).to.equal(true)
expect(WorkerMock.instances).to.have.length(2)
})
it('sends init to the new worker once it reports listening after reset', function () {
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
fileUploader: fileUploaderStub,
})
const originalWorker = WorkerMock.instances[0]
originalWorker.emitMessage({ type: 'listening' })
originalWorker.emitMessage({ type: 'loaded' })
client.reset()
const newWorker = WorkerMock.instances[1]
expect(newWorker.postedMessages).to.have.length(0)
newWorker.emitMessage({ type: 'listening' })
expect(newWorker.postedMessages).to.deep.equal([
{
type: 'init',
baseAssetPath: BASE_ASSET_PATH,
},
])
})
it('allows running code on the new worker after reset', function () {
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
fileUploader: fileUploaderStub,
})
const originalWorker = WorkerMock.instances[0]
originalWorker.emitMessage({ type: 'listening' })
originalWorker.emitMessage({ type: 'loaded' })
client.reset()
const newWorker = WorkerMock.instances[1]
newWorker.emitMessage({ type: 'listening' })
newWorker.emitMessage({ type: 'loaded' })
client.runCode('print("after reset")', {
fileId: 'main.py',
executionId: 'exec-5',
files: [],
})
const runRequest = newWorker.postedMessages.find(
(message: any) => message.type === 'run-code'
)
expect(runRequest).to.include({
type: 'run-code',
fileId: 'main.py',
executionId: 'exec-5',
code: 'print("after reset")',
})
})
it('reset is a no-op after destroy', function () {
const client = new PyodideWorkerClient({
baseAssetPath: BASE_ASSET_PATH,
createWorker,
fileUploader: fileUploaderStub,
})
const originalWorker = WorkerMock.instances[0]
originalWorker.emitMessage({ type: 'listening' })
originalWorker.emitMessage({ type: 'loaded' })
client.destroy()
client.reset()
// No new worker should have been created
expect(WorkerMock.instances).to.have.length(1)
})
})
})
services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-output-limits.spec.ts
-93
View File
@@ -1,93 +0,0 @@
import { expect } from 'chai'
import {
MAX_OUTPUT_FILES,
MAX_OUTPUT_FILE_BYTES,
MAX_OUTPUT_TOTAL_BYTES,
checkOutputLimits,
} from '@/features/ide-react/components/editor/python/pyodide-worker-output-limits'
const BYTES_PER_MB = 1024 * 1024
function makeFiles(
count: number,
sizePerFile: number
): { path: string; size: number }[] {
return Array.from({ length: count }, (_, i) => ({
path: `/project/file${i}.bin`,
size: sizePerFile,
}))
}
describe('checkOutputLimits', function () {
it('returns null when both limits are within bounds', function () {
const files = makeFiles(10, 1024)
expect(checkOutputLimits(files)).to.equal(null)
})
it('returns null for an empty file list', function () {
expect(checkOutputLimits([])).to.equal(null)
})
it('returns null at exactly the file count limit', function () {
const files = makeFiles(MAX_OUTPUT_FILES, 1024)
expect(checkOutputLimits(files)).to.equal(null)
})
it('returns a count violation reporting the actual file count when the file count exceeds the limit', function () {
const files = makeFiles(73, 1)
const violation = checkOutputLimits(files)
expect(violation).to.deep.equal({
kind: 'count',
message: 'Output limit exceeded: 73 files generated (max 50)',
})
})
it('returns null at exactly the per-file size limit', function () {
const files = [{ path: '/project/big.bin', size: MAX_OUTPUT_FILE_BYTES }]
expect(checkOutputLimits(files)).to.equal(null)
})
it('returns a single-file-size violation reporting the offending file path and size when one file exceeds the per-file limit', function () {
const files = [
{ path: '/project/small.bin', size: 1 * BYTES_PER_MB },
{ path: '/project/huge.bin', size: 80 * BYTES_PER_MB },
]
const violation = checkOutputLimits(files)
expect(violation).to.deep.equal({
kind: 'single-file-size',
message:
'Output limit exceeded: /project/huge.bin is 80MB (max 50MB per file)',
})
})
it('returns a total-output-size violation when the summed size exceeds the total limit', function () {
const files = [
{ path: '/project/a.bin', size: 40 * BYTES_PER_MB },
{ path: '/project/b.bin', size: 40 * BYTES_PER_MB },
{ path: '/project/c.bin', size: 40 * BYTES_PER_MB },
]
const violation = checkOutputLimits(files)
expect(violation).to.not.equal(null)
expect(violation!.kind).to.equal('total-output-size')
expect(violation!.message).to.equal(
'Output limit exceeded: 120MB total (max 100MB)'
)
})
it('returns null at exactly the total size limit', function () {
const files = [
{ path: '/project/a.bin', size: 50 * BYTES_PER_MB },
{ path: '/project/b.bin', size: 50 * BYTES_PER_MB },
]
expect(checkOutputLimits(files)).to.equal(null)
const total = files.reduce((acc, f) => acc + f.size, 0)
expect(total).to.equal(MAX_OUTPUT_TOTAL_BYTES)
})
it('reports the count violation when both limits are exceeded', function () {
const files = makeFiles(MAX_OUTPUT_FILES + 10, MAX_OUTPUT_TOTAL_BYTES)
const violation = checkOutputLimits(files)
expect(violation).to.not.equal(null)
expect(violation!.kind).to.equal('count')
})
})
services/web/test/frontend/features/ide-react/unit/editor/python-runner.spec.ts
-499
View File
@@ -1,499 +0,0 @@
import { expect } from 'chai'
import sinon from 'sinon'
import {
PythonRunner,
PythonRunnerState,
DEFAULT_STATE,
ExecutionContext,
type FileUploader,
} from '@/features/ide-react/components/editor/python/python-runner'
import { WorkerMock, createWorker } from './worker-mock'
const BASE_ASSET_PATH = 'https://assets.example.test/'
const FILE_ID = 'file-1'
function createRunner(
overrides: {
fileId?: string
getExecutionContext?: () => Promise<ExecutionContext | null>
fileUploader?: FileUploader
} = {}
) {
const fileId = overrides.fileId ?? FILE_ID
const getExecutionContext =
overrides.getExecutionContext ??
(() =>
Promise.resolve({
code: 'print("hello")',
files: [{ relativePath: 'main.py', content: 'print("hello")' }],
}))
const fileUploader = overrides.fileUploader ?? sinon.stub().resolves([])
const runner = new PythonRunner(
fileId,
BASE_ASSET_PATH,
getExecutionContext,
createWorker,
fileUploader
)
return runner
}
function initAndLoad(runner: PythonRunner) {
runner.init()
const worker = WorkerMock.instances[WorkerMock.instances.length - 1]
worker.emitMessage({ type: 'listening' })
worker.emitMessage({ type: 'loaded' })
return worker
}
function waitForState(
runner: PythonRunner,
predicate: (state: PythonRunnerState) => boolean
): Promise<PythonRunnerState> {
return new Promise(resolve => {
if (predicate(runner.getState())) {
resolve(runner.getState())
return
}
const unsubscribe = runner.subscribe(() => {
if (predicate(runner.getState())) {
unsubscribe()
resolve(runner.getState())
}
})
})
}
describe('PythonRunner', function () {
beforeEach(function () {
WorkerMock.instances.length = 0
})
describe('initial state', function () {
it('starts with default snapshot before init', function () {
const runner = createRunner()
expect(runner.getState()).to.deep.equal(DEFAULT_STATE)
})
})
describe('init and lifecycle', function () {
it('transitions to loading on init', function () {
const runner = createRunner()
runner.init()
expect(runner.getState().status).to.equal('loading')
})
it('transitions to idle when worker reports loaded', function () {
const runner = createRunner()
initAndLoad(runner)
expect(runner.getState().status).to.equal('idle')
})
it('transitions to errored on loading failure', function () {
const runner = createRunner()
runner.init()
const worker = WorkerMock.instances[0]
worker.emitMessage({ type: 'listening' })
worker.emitMessage({
type: 'loading-failed',
error: 'network error',
})
expect(runner.getState().status).to.equal('errored')
expect(runner.getState().error).to.equal('network error')
})
it('clears error on successful load after failure', function () {
const runner = createRunner()
runner.init()
const worker = WorkerMock.instances[0]
worker.emitMessage({ type: 'listening' })
worker.emitMessage({ type: 'loaded' })
expect(runner.getState().error).to.equal(null)
})
it('is a no-op if already initialized', function () {
const runner = createRunner()
runner.init()
runner.init()
expect(WorkerMock.instances).to.have.length(1)
})
})
describe('run', function () {
it('transitions to running then finished', async function () {
const runner = createRunner()
const worker = initAndLoad(runner)
await runner.run()
expect(runner.getState().status).to.equal('running')
const runMsg = worker.postedMessages.find(m => m.type === 'run-code')
worker.emitMessage({
type: 'run-code-result',
fileId: FILE_ID,
executionId: runMsg.executionId,
success: true,
outputs: [],
outputFiles: [],
imports: [],
})
await waitForState(runner, s => s.status === 'finished')
expect(runner.getState().status).to.equal('finished')
})
it('clears previous output on new run', async function () {
const runner = createRunner()
const worker = initAndLoad(runner)
await runner.run()
const runMsg = worker.postedMessages.find(m => m.type === 'run-code')
worker.emitMessage({
type: 'output-line',
stream: 'stdout',
line: 'first run output',
fileId: FILE_ID,
executionId: runMsg.executionId,
})
worker.emitMessage({
type: 'run-code-result',
fileId: FILE_ID,
executionId: runMsg.executionId,
success: true,
outputs: [],
outputFiles: [],
imports: [],
})
expect(runner.getState().output).to.deep.equal([
{ stream: 'stdout', line: 'first run output' },
])
await runner.run()
expect(runner.getState().output).to.deep.equal([])
})
it('is a no-op while still loading', async function () {
const runner = createRunner()
runner.init()
await runner.run()
expect(runner.getState().status).to.equal('loading')
})
it('is a no-op when getExecutionContext returns null', async function () {
const runner = createRunner({
getExecutionContext: () => Promise.resolve(null),
})
initAndLoad(runner)
await runner.run()
expect(runner.getState().status).to.equal('idle')
})
it('transitions to errored when getExecutionContext rejects', async function () {
const runner = createRunner({
getExecutionContext: () => Promise.reject(new Error('network failure')),
})
initAndLoad(runner)
await runner.run()
expect(runner.getState().status).to.equal('errored')
expect(runner.getState().error).to.equal('network failure')
})
})
describe('files-saved toast', function () {
let toastEvents: CustomEvent[]
let toastListener: (event: Event) => void
beforeEach(function () {
toastEvents = []
toastListener = event => {
toastEvents.push(event as CustomEvent)
}
window.addEventListener('ide:show-toast', toastListener)
})
afterEach(function () {
window.removeEventListener('ide:show-toast', toastListener)
})
it('dispatches a files-saved toast with successfully uploaded paths', async function () {
const runner = createRunner()
const worker = initAndLoad(runner)
await runner.run()
const runMsg = worker.postedMessages.find(m => m.type === 'run-code')
worker.emitMessage({
type: 'run-code-result',
fileId: FILE_ID,
executionId: runMsg.executionId,
success: true,
outputs: ['/project/foo.txt', '/project/bar.csv'],
outputFiles: [],
imports: [],
failedUploads: [],
})
await waitForState(runner, s => s.status === 'finished')
expect(toastEvents).to.have.length(1)
expect(toastEvents[0].detail).to.deep.equal({
key: 'python:files-saved',
paths: ['foo.txt', 'bar.csv'],
})
})
it('excludes failed uploads from the toast', async function () {
const fileUploader = sinon.stub().resolves([
{ status: 'success', name: 'foo.txt', relativePath: 'foo.txt' },
{
status: 'error',
name: 'bar.csv',
relativePath: 'bar.csv',
error: 'boom',
},
])
const runner = createRunner({ fileUploader })
const worker = initAndLoad(runner)
await runner.run()
const runMsg = worker.postedMessages.find(m => m.type === 'run-code')
worker.emitMessage({
type: 'run-code-result',
fileId: FILE_ID,
executionId: runMsg.executionId,
success: true,
outputs: ['/project/foo.txt', '/project/bar.csv'],
outputFiles: [
{ relativePath: 'foo.txt', content: new Uint8Array() },
{ relativePath: 'bar.csv', content: new Uint8Array() },
],
imports: [],
})
await waitForState(runner, s => s.status === 'finished')
expect(toastEvents).to.have.length(1)
expect(toastEvents[0].detail).to.deep.equal({
key: 'python:files-saved',
paths: ['foo.txt'],
})
})
it('does not dispatch a toast when no outputs were written', async function () {
const runner = createRunner()
const worker = initAndLoad(runner)
await runner.run()
const runMsg = worker.postedMessages.find(m => m.type === 'run-code')
worker.emitMessage({
type: 'run-code-result',
fileId: FILE_ID,
executionId: runMsg.executionId,
success: true,
outputs: [],
outputFiles: [],
imports: [],
failedUploads: [],
})
await waitForState(runner, s => s.status === 'finished')
expect(toastEvents).to.have.length(0)
})
it('does not dispatch a toast when every output failed to upload', async function () {
const fileUploader = sinon.stub().resolves([
{
status: 'error',
name: 'foo.txt',
relativePath: 'foo.txt',
error: 'boom',
},
])
const runner = createRunner({ fileUploader })
const worker = initAndLoad(runner)
await runner.run()
const runMsg = worker.postedMessages.find(m => m.type === 'run-code')
worker.emitMessage({
type: 'run-code-result',
fileId: FILE_ID,
executionId: runMsg.executionId,
success: true,
outputs: ['/project/foo.txt'],
outputFiles: [{ relativePath: 'foo.txt', content: new Uint8Array() }],
imports: [],
})
await waitForState(runner, s => s.status === 'finished')
expect(toastEvents).to.have.length(0)
})
})
describe('output', function () {
it('accumulates output lines for the matching file', async function () {
const runner = createRunner()
const worker = initAndLoad(runner)
await runner.run()
const runMsg = worker.postedMessages.find(m => m.type === 'run-code')
worker.emitMessage({
type: 'output-line',
stream: 'stdout',
line: 'line 1',
fileId: FILE_ID,
executionId: runMsg.executionId,
})
worker.emitMessage({
type: 'output-line',
stream: 'stderr',
line: 'line 2',
fileId: FILE_ID,
executionId: runMsg.executionId,
})
expect(runner.getState().output).to.deep.equal([
{ stream: 'stdout', line: 'line 1' },
{ stream: 'stderr', line: 'line 2' },
])
})
it('ignores output for a different fileId', async function () {
const runner = createRunner()
const worker = initAndLoad(runner)
await runner.run()
const runMsg = worker.postedMessages.find(m => m.type === 'run-code')
worker.emitMessage({
type: 'output-line',
stream: 'stdout',
line: 'other file output',
fileId: 'different-file',
executionId: runMsg.executionId,
})
expect(runner.getState().output).to.deep.equal([])
})
it('ignores output for a stale executionId', async function () {
const runner = createRunner()
const worker = initAndLoad(runner)
await runner.run()
worker.emitMessage({
type: 'output-line',
stream: 'stdout',
line: 'stale output',
fileId: FILE_ID,
executionId: 'old-execution-id',
})
expect(runner.getState().output).to.deep.equal([])
})
it('caps output at 100 lines', async function () {
const runner = createRunner()
const worker = initAndLoad(runner)
await runner.run()
const runMsg = worker.postedMessages.find(m => m.type === 'run-code')
for (let i = 0; i < 110; i++) {
worker.emitMessage({
type: 'output-line',
stream: 'stdout',
line: `line ${i}`,
fileId: FILE_ID,
executionId: runMsg.executionId,
})
}
const output = runner.getState().output
expect(output).to.have.length(100)
expect(output[0]).to.deep.equal({ stream: 'stdout', line: 'line 10' })
expect(output[99]).to.deep.equal({ stream: 'stdout', line: 'line 109' })
})
})
describe('interrupt', function () {
it('appends interrupted message and transitions to loading when running', async function () {
const runner = createRunner()
const worker = initAndLoad(runner)
await runner.run()
const runMsg = worker.postedMessages.find(m => m.type === 'run-code')
worker.emitMessage({
type: 'output-line',
stream: 'stdout',
line: 'partial output',
fileId: FILE_ID,
executionId: runMsg.executionId,
})
runner.interrupt()
expect(runner.getState().status).to.equal('loading')
expect(runner.getState().output).to.deep.equal([
{ stream: 'stdout', line: 'partial output' },
{ stream: 'info', line: 'Execution interrupted' },
])
})
it('does not append interrupted message when not running', function () {
const runner = createRunner()
initAndLoad(runner)
runner.interrupt()
expect(runner.getState().status).to.equal('loading')
expect(runner.getState().output).to.deep.equal([])
})
})
describe('subscribe', function () {
it('notifies listeners on state changes', function () {
const runner = createRunner()
const listener = sinon.stub()
runner.subscribe(listener)
initAndLoad(runner)
expect(listener.callCount).to.be.greaterThan(0)
})
it('stops notifying after unsubscribe', function () {
const runner = createRunner()
const listener = sinon.stub()
const unsubscribe = runner.subscribe(listener)
runner.init()
const countAfterInit = listener.callCount
unsubscribe()
const worker = WorkerMock.instances[0]
worker.emitMessage({ type: 'listening' })
worker.emitMessage({ type: 'loaded' })
expect(listener.callCount).to.equal(countAfterInit)
})
})
describe('destroy', function () {
it('terminates the worker', function () {
const runner = createRunner()
initAndLoad(runner)
runner.destroy()
const worker = WorkerMock.instances[0]
expect(worker.terminated).to.equal(true)
})
})
})
services/web/test/frontend/features/ide-react/unit/editor/worker-mock.ts
-35
View File
@@ -1,35 +0,0 @@
type WorkerMessageListener = (event: MessageEvent) => void
export class WorkerMock {
static instances: WorkerMock[] = []
readonly postedMessages: any[] = []
terminated = false
private messageListeners: WorkerMessageListener[] = []
constructor() {
WorkerMock.instances.push(this)
}
addEventListener(type: string, listener: WorkerMessageListener) {
if (type === 'message') {
this.messageListeners.push(listener)
}
}
postMessage(message: unknown) {
this.postedMessages.push(message)
}
terminate() {
this.terminated = true
}
emitMessage(message: unknown) {
for (const listener of this.messageListeners) {
listener({ data: message, target: this } as unknown as MessageEvent)
}
}
}
export const createWorker = () => new WorkerMock() as unknown as Worker