diff --git a/package-lock.json b/package-lock.json index c9a568aaf5..98b089a8ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4602,6 +4602,20 @@ "@lezer/markdown": "^1.0.0" } }, + "node_modules/@codemirror/lang-python": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/@codemirror/lang-python/-/lang-python-6.2.1.tgz", + "integrity": "sha512-IRjC8RUBhn9mGR9ywecNhB51yePWCGgvHfY1lWN/Mrp3cKuHr0isDKia+9HnvhiWNnMpbGhWrkhuWOc09exRyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.3.2", + "@codemirror/language": "^6.8.0", + "@codemirror/state": "^6.0.0", + "@lezer/common": "^1.2.1", + "@lezer/python": "^1.1.4" + } + }, "node_modules/@codemirror/language": { "version": "6.12.1", "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.12.1.tgz", @@ -9056,6 +9070,18 @@ "@lezer/highlight": "^1.0.0" } }, + "node_modules/@lezer/python": { + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/@lezer/python/-/python-1.1.18.tgz", + "integrity": "sha512-31FiUrU7z9+d/ElGQLJFXl+dKOdx0jALlP3KEOsGTex8mvj+SoE1FgItcHWK/axkxCHGUSpqIHt6JAWfWu9Rhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0" + } + }, "node_modules/@lit-labs/react": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@lit-labs/react/-/react-1.2.1.tgz", @@ -16044,6 +16070,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/eslint": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-9.6.1.tgz", @@ -39596,6 +39629,20 @@ "node": ">=6" } }, + "node_modules/pyodide": { + "version": "0.29.3", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-0.29.3.tgz", + "integrity": "sha512-22UBuhOJawj7vKUnS7/F3xK+515LJdjiMAHoCfuS6/PbHiOrSQVnYwDe+2sbVwiOZ3sMMexdXICew6NqOMQGgA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/qified": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/qified/-/qified-0.6.0.tgz", @@ -51152,6 +51199,7 @@ "@codemirror/autocomplete": "github:overleaf/codemirror-autocomplete#6445cd056671c98d12d1c597ba705e11327ec4c5", "@codemirror/commands": "6.10.1", "@codemirror/lang-markdown": "6.5.0", + "@codemirror/lang-python": "6.2.1", "@codemirror/language": "6.12.1", "@codemirror/lint": "6.9.2", "@codemirror/search": "github:overleaf/codemirror-search#04380a528c339cd4b78fb10b3ef017f657ec17bd", @@ -51315,6 +51363,7 @@ "postcss": "^8.4.31", "postcss-loader": "^7.3.3", "prop-types": "^15.7.2", + "pyodide": "^0.29.0", "qrcode": "^1.4.4", "react": "^18.3.1", "react-bootstrap": "^2.10.10", diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts new file mode 100644 index 0000000000..e334177f41 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts @@ -0,0 +1,148 @@ +import type { + ProjectFileData, + PyodideWorkerRequest, + PyodideWorkerResponse, +} from './pyodide-worker-messages' + +export type OutputCallback = ( + stream: 'stdout' | 'stderr', + line: string, + requestId?: string +) => void + +export type LifecycleCallback = ( + event: + | { type: 'loaded' } + | { type: 'loading-failed'; error: string } + | { type: 'run-finished'; requestId: string } +) => void + +export class PyodideWorkerClient { + private worker: Worker + private listening = false + private loaded = false + private destroyed = false + private loadingError: string | null = null + private pendingMessages: PyodideWorkerRequest[] = [] + private outputCallback: OutputCallback | null = null + private lifecycleCallback: LifecycleCallback | null = null + + constructor(options: { baseAssetPath: string }) { + const { baseAssetPath } = options + + this.worker = new Worker( + /* webpackChunkName: "pyodide-worker" */ + new URL('./pyodide.worker.ts', import.meta.url), + { type: 'module' } + ) + + this.worker.addEventListener('message', this.receive.bind(this)) + + this.queueMessage({ + type: 'init', + baseAssetPath, + }) + } + + setOutputCallback(callback: OutputCallback) { + this.outputCallback = callback + } + + setLifecycleCallback(handler: LifecycleCallback) { + this.lifecycleCallback = handler + + if (this.loaded) { + handler({ type: 'loaded' }) + return + } + if (this.loadingError) { + handler({ type: 'loading-failed', error: this.loadingError }) + } + } + + runCode( + code: string, + options: { requestId: string; files: ProjectFileData[] } + ): void { + if (this.destroyed) { + throw new Error('Pyodide worker client has been destroyed') + } + + if (this.loadingError) { + throw new Error(this.loadingError) + } + + this.queueMessage({ + type: 'run-code', + code, + id: options.requestId, + files: options.files, + }) + } + + destroy() { + if (this.destroyed) { + return + } + + this.destroyed = true + this.pendingMessages.length = 0 + + if (!this.loaded && !this.loadingError) { + this.loadingError = 'Pyodide worker was destroyed before loading finished' + } + + this.worker.terminate() + } + + private queueMessage(message: PyodideWorkerRequest) { + if (this.listening) { + this.worker.postMessage(message) + } else { + this.pendingMessages.push(message) + } + } + + private receive(event: MessageEvent) { + const response = event.data + + switch (response.type) { + case 'listening': + this.listening = true + for (const message of this.pendingMessages) { + this.worker.postMessage(message) + } + this.pendingMessages.length = 0 + return + + case 'loaded': + this.loaded = true + this.lifecycleCallback?.({ type: 'loaded' }) + return + + case 'loading-failed': + this.loadingError = response.error + this.pendingMessages.length = 0 + this.lifecycleCallback?.({ + type: 'loading-failed', + error: response.error, + }) + return + + case 'output-line': + this.outputCallback?.( + response.stream, + response.line, + response.requestId + ) + break + + case 'run-code-result': + this.lifecycleCallback?.({ + type: 'run-finished', + requestId: response.id, + }) + break + } + } +} diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts new file mode 100644 index 0000000000..2512531dca --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts @@ -0,0 +1,48 @@ +export type ProjectFileData = { + relativePath: string + content: string +} + +// Main thread -> Worker messages + +export type InitRequest = { + type: 'init' + baseAssetPath: string +} + +export type RunCodeRequest = { + type: 'run-code' + id: string + code: string + files: ProjectFileData[] +} + +export type PyodideWorkerRequest = InitRequest | RunCodeRequest + +// Worker -> Main thread lifecycle and streaming events + +export type ListeningEvent = { type: 'listening' } +export type LoadedEvent = { type: 'loaded' } +export type LoadingFailedEvent = { type: 'loading-failed'; error: string } + +export type OutputLineEvent = { + type: 'output-line' + stream: 'stdout' | 'stderr' + line: string + requestId?: string +} + +export type PyodideWorkerEvent = + | ListeningEvent + | LoadedEvent + | LoadingFailedEvent + | OutputLineEvent + +// Worker -> Main thread ID responses + +export type RunCodeResult = { + type: 'run-code-result' + id: string +} + +export type PyodideWorkerResponse = PyodideWorkerEvent | RunCodeResult diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts new file mode 100644 index 0000000000..3569399e9a --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts @@ -0,0 +1,216 @@ +import path from 'path-browserify' +import type { + ProjectFileData, + PyodideWorkerRequest, + RunCodeRequest, +} from './pyodide-worker-messages' + +type PyodideRuntimeModule = { + loadPyodide: (options: { + indexURL: string + packageBaseUrl?: string + env?: Record + packages?: string[] + }) => Promise +} + +type PyodideInstance = { + FS: unknown + runPythonAsync: (code: string) => Promise + setStdout: (options: { batched: (line: string) => void }) => void + setStderr: (options: { batched: (line: string) => void }) => void +} + +type PyodideFs = { + analyzePath: (filePath: string) => { exists: boolean } + mkdir: (filePath: string) => void + writeFile: ( + filePath: string, + content: string | ArrayBuffer | Uint8Array + ) => void + chdir: (filePath: string) => void +} + +const PROJECT_FS_ROOT = '/project' +const PYODIDE_INDEX_PATH = 'js/libs/pyodide/' + +function getPyodideIndexUrl(baseAssetPath: string): string { + return new URL(PYODIDE_INDEX_PATH, baseAssetPath).toString() +} + +function toRuntimeProjectPath(relativePath: string): string { + return path.posix.join(PROJECT_FS_ROOT, path.posix.normalize(relativePath)) +} + +function ensureProjectRootExists(fs: PyodideFs) { + try { + const projectRootAnalysis = fs.analyzePath(PROJECT_FS_ROOT) + if (!projectRootAnalysis.exists) { + fs.mkdir(PROJECT_FS_ROOT) + } + } catch { + fs.mkdir(PROJECT_FS_ROOT) + } +} + +function ensureDirectoryExists(fs: PyodideFs, filePath: string) { + const directory = path.dirname(filePath) + if (directory === '.' || directory === '/') { + return + } + + let currentPath = directory.startsWith('/') ? '/' : '' + for (const part of directory.split('/').filter(Boolean)) { + currentPath = path.posix.join(currentPath, part) + try { + const analysis = fs.analyzePath(currentPath) + if (!analysis.exists) { + fs.mkdir(currentPath) + } + } catch { + // Ignore failures when a directory already exists. + } + } +} + +function syncProjectFiles(fs: PyodideFs, files: ProjectFileData[]) { + ensureProjectRootExists(fs) + + for (const file of files) { + const runtimePath = toRuntimeProjectPath(file.relativePath) + ensureDirectoryExists(fs, runtimePath) + fs.writeFile(runtimePath, file.content) + } + + fs.chdir(PROJECT_FS_ROOT) +} + +let pyodideInstance: PyodideInstance | null = null +let activeRunRequestId: string | null = null + +async function loadPyodideModule( + pyodideIndexUrl: string +): Promise { + const runtimeModuleUrl = `${pyodideIndexUrl}pyodide.mjs` + + try { + return (await import( + /* webpackIgnore: true */ runtimeModuleUrl + )) as PyodideRuntimeModule + } catch (loadError) { + const loadErrorMessage = + loadError instanceof Error ? loadError.message : String(loadError) + throw new Error( + `Unable to load Pyodide module from ${runtimeModuleUrl}. Original error: ${loadErrorMessage}` + ) + } +} + +async function handleInit(msg: { baseAssetPath: string }) { + const pyodideIndexUrl = getPyodideIndexUrl(msg.baseAssetPath) + + try { + const pyodideModule = await loadPyodideModule(pyodideIndexUrl) + const instance = await pyodideModule.loadPyodide({ + indexURL: pyodideIndexUrl, + packageBaseUrl: pyodideIndexUrl, + env: { MPLBACKEND: 'Agg' }, + }) + + instance.setStdout({ + batched: (line: string) => { + self.postMessage({ + type: 'output-line', + stream: 'stdout', + line, + requestId: activeRunRequestId ?? undefined, + }) + }, + }) + instance.setStderr({ + batched: (line: string) => { + self.postMessage({ + type: 'output-line', + stream: 'stderr', + line, + requestId: activeRunRequestId ?? undefined, + }) + }, + }) + + pyodideInstance = instance + self.postMessage({ type: 'loaded' }) + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error) + console.error('Pyodide initialization failed', error) + self.postMessage({ + type: 'loading-failed', + error: errorMessage, + }) + } +} + +async function handleRunCode(msg: RunCodeRequest) { + if (!pyodideInstance) { + const error = 'Pyodide is not initialized' + self.postMessage({ + type: 'output-line', + stream: 'stderr', + line: error, + requestId: msg.id, + }) + self.postMessage({ + type: 'run-code-result', + id: msg.id, + }) + return + } + + activeRunRequestId = msg.id + try { + if (msg.files.length > 0) { + const fs = pyodideInstance.FS as PyodideFs + syncProjectFiles(fs, msg.files) + } + + const result = await pyodideInstance.runPythonAsync(msg.code) + if (result !== undefined) { + self.postMessage({ + type: 'output-line', + stream: 'stdout', + line: String(result), + requestId: activeRunRequestId ?? undefined, + }) + } + } catch (runError) { + const errorMessage = + runError instanceof Error ? runError.message : String(runError) + self.postMessage({ + type: 'output-line', + stream: 'stderr', + line: errorMessage, + requestId: activeRunRequestId ?? undefined, + }) + } finally { + activeRunRequestId = null + } + + self.postMessage({ + type: 'run-code-result', + id: msg.id, + }) +} + +self.addEventListener('message', async event => { + const msg = event.data as PyodideWorkerRequest + switch (msg.type) { + case 'init': + await handleInit(msg) + break + case 'run-code': + await handleRunCode(msg) + break + } +}) + +self.postMessage({ type: 'listening' }) diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/python-output-pane.tsx b/services/web/frontend/js/features/ide-react/components/editor/python/python-output-pane.tsx new file mode 100644 index 0000000000..879b5c3a16 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/editor/python/python-output-pane.tsx @@ -0,0 +1,189 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import path from 'path-browserify' +import OLButton from '@/shared/components/ol/ol-button' +import OLButtonToolbar from '@/shared/components/ol/ol-button-toolbar' +import MaterialIcon from '@/shared/components/material-icon' +import { useEditorOpenDocContext } from '@/features/ide-react/context/editor-open-doc-context' +import { useFileTreePathContext } from '@/features/file-tree/contexts/file-tree-path' +import { debugConsole } from '@/utils/debugging' +import getMeta from '@/utils/meta' +import { PyodideWorkerClient } from './pyodide-worker-client' + +export default function PythonOutputPane() { + const { currentDocument, currentDocumentId } = useEditorOpenDocContext() + const { pathInFolder } = useFileTreePathContext() + const clientRef = useRef(null) + const currentRequestIdRef = useRef(null) + const [isReady, setIsReady] = useState(false) + const [output, setOutput] = useState([]) + const [isRunning, setIsRunning] = useState(false) + const [isLoadingPyodide, setIsLoadingPyodide] = useState(true) + const [error, setError] = useState(null) + + const appendOutput = useCallback((line: string) => { + setOutput(previousOutput => [...previousOutput, line]) + }, []) + + useEffect(() => { + const baseAssetPath = new URL( + getMeta('ol-baseAssetPath'), + window.location.href + ).toString() + const client = new PyodideWorkerClient({ baseAssetPath }) + clientRef.current = client + let cancelled = false + + client.setLifecycleCallback(event => { + if (cancelled) { + return + } + + switch (event.type) { + case 'loaded': + setIsReady(true) + setIsLoadingPyodide(false) + return + + case 'loading-failed': + debugConsole.error('Failed to load Python runtime', event.error) + setError(formatError(event.error)) + setIsLoadingPyodide(false) + setIsRunning(false) + return + + case 'run-finished': + if (event.requestId !== currentRequestIdRef.current) { + return + } + currentRequestIdRef.current = null + setIsRunning(false) + break + } + }) + + client.setOutputCallback((_stream, line, requestId) => { + if (!requestId || requestId !== currentRequestIdRef.current) { + return + } + appendOutput(line) + }) + + return () => { + cancelled = true + currentRequestIdRef.current = null + client.destroy() + clientRef.current = null + } + }, [appendOutput]) + + useEffect(() => { + currentRequestIdRef.current = null + setIsRunning(false) + setOutput([]) + setError(null) + }, [currentDocumentId]) + + const buildCurrentDocumentSyncFile = useCallback(() => { + if (!currentDocument || !currentDocumentId) { + return null + } + + const content = currentDocument.getSnapshot() + if (typeof content !== 'string') { + return null + } + + const currentPath = pathInFolder(currentDocumentId) + if (!currentPath) { + throw new Error( + 'Unable to resolve current document path for Python sync.' + ) + } + + return { + relativePath: path.posix.normalize(currentPath), + content, + } + }, [currentDocument, currentDocumentId, pathInFolder]) + + const handleRun = useCallback(() => { + const client = clientRef.current + if (!client || !isReady) { + return + } + + const syncFile = buildCurrentDocumentSyncFile() + if (!syncFile) { + return + } + + setOutput([]) + setError(null) + + const requestId = syncFile.relativePath + currentRequestIdRef.current = requestId + setIsRunning(true) + + try { + client.runCode(syncFile.content, { requestId, files: [syncFile] }) + } catch (runError) { + if (currentRequestIdRef.current !== requestId) { + return + } + currentRequestIdRef.current = null + setIsRunning(false) + setError(formatError(runError)) + } + }, [buildCurrentDocumentSyncFile, isReady]) + + return ( +
+ +
+
+ + {isRunning ? 'Running...' : 'Run'} + + +
+
+ + +
+ {isLoadingPyodide && ( +
+ Loading Python runtime... +
+ )} + {!isLoadingPyodide && !error && output.length === 0 && ( +
+ Run the current script to see output. +
+ )} + {error && ( +
{error}
+ )} + {output.map((line, index) => ( +
+ {line} +
+ ))} +
+
+ ) +} + +function formatError(error: unknown): string { + if (error instanceof Error) { + return error.message + } + + return String(error) +} diff --git a/services/web/frontend/js/features/ide-react/components/layout/editor.tsx b/services/web/frontend/js/features/ide-react/components/layout/editor.tsx index 9b5e83cbbb..deab008812 100644 --- a/services/web/frontend/js/features/ide-react/components/layout/editor.tsx +++ b/services/web/frontend/js/features/ide-react/components/layout/editor.tsx @@ -9,12 +9,16 @@ import { Suspense } from 'react' import { FullSizeLoadingSpinner } from '@/shared/components/loading-spinner' import SymbolPalettePane from '@/features/ide-react/components/editor/symbol-palette-pane' import { useEditorPropertiesContext } from '@/features/ide-react/context/editor-properties-context' +import { PythonEditorSplit } from '@/features/ide-react/components/layout/python-editor-split' export const Editor = () => { const { opening, errorState, showSymbolPalette } = useEditorPropertiesContext() const { selectedEntityCount, openEntity } = useFileTreeOpenContext() const { currentDocumentId, currentDocument } = useEditorOpenDocContext() + const isPythonDocument = + openEntity?.type === 'doc' && + openEntity.entity.name.toLowerCase().endsWith('.py') if (!currentDocumentId) { return null @@ -39,7 +43,7 @@ export const Editor = () => { order={1} className="ide-redesign-editor-panel" > - + {isPythonDocument ? : } {isLoading && } {showSymbolPalette && ( diff --git a/services/web/frontend/js/features/ide-react/components/layout/python-editor-split.tsx b/services/web/frontend/js/features/ide-react/components/layout/python-editor-split.tsx new file mode 100644 index 0000000000..a78c4feeb2 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/layout/python-editor-split.tsx @@ -0,0 +1,27 @@ +import { Panel, PanelGroup } from 'react-resizable-panels' +import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle' +import PythonOutputPane from '@/features/ide-react/components/editor/python/python-output-pane' +import SourceEditor from '@/features/source-editor/components/source-editor' + +export const PythonEditorSplit = () => { + return ( + + + + + + + + + + ) +} diff --git a/services/web/frontend/js/features/source-editor/languages/index.ts b/services/web/frontend/js/features/source-editor/languages/index.ts index dd37dbc507..9a698b056a 100644 --- a/services/web/frontend/js/features/source-editor/languages/index.ts +++ b/services/web/frontend/js/features/source-editor/languages/index.ts @@ -65,4 +65,11 @@ export const languages = [ return import('./markdown').then(m => m.markdown()) }, }), + LanguageDescription.of({ + name: 'python', + extensions: ['py'], + load: () => { + return import('@codemirror/lang-python').then(m => m.python()) + }, + }), ] diff --git a/services/web/frontend/stylesheets/pages/editor/ide-redesign.scss b/services/web/frontend/stylesheets/pages/editor/ide-redesign.scss index 2fac37ca13..f684b867ca 100644 --- a/services/web/frontend/stylesheets/pages/editor/ide-redesign.scss +++ b/services/web/frontend/stylesheets/pages/editor/ide-redesign.scss @@ -54,6 +54,47 @@ height: 100%; } +.ide-redesign-python-editor-split { + height: 100%; +} + +.ide-redesign-python-output-pane { + height: 100%; + display: flex; + flex-direction: column; + background-color: var(--ide-redesign-background); + color: var(--ide-redesign-color); + border-top: 1px solid var(--border-divider); +} + +.ide-redesign-python-output-pane-body { + flex: 1; + overflow: auto; + padding: var(--spacing-06); + font-family: 'DM Mono', monospace; + font-size: var(--font-size-02); + line-height: var(--line-height-03); +} + +.ide-redesign-python-output-pane-placeholder { + color: var(--content-secondary); +} + +.ide-redesign-python-output-pane-line { + white-space: pre-wrap; + word-break: break-word; +} + +.ide-redesign-python-output-pane-placeholder, +.ide-redesign-python-output-pane-error { + white-space: pre-wrap; +} + +.ide-redesign-python-output-pane-error { + margin-bottom: var(--spacing-03); + color: var(--red-50); +} + .ide-redesign-labs-user-beta-promo { position: absolute; top: 60px; diff --git a/services/web/package.json b/services/web/package.json index 9c9492aeed..3326545e62 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -205,6 +205,7 @@ "@codemirror/autocomplete": "github:overleaf/codemirror-autocomplete#6445cd056671c98d12d1c597ba705e11327ec4c5", "@codemirror/commands": "6.10.1", "@codemirror/lang-markdown": "6.5.0", + "@codemirror/lang-python": "6.2.1", "@codemirror/language": "6.12.1", "@codemirror/lint": "6.9.2", "@codemirror/search": "github:overleaf/codemirror-search#04380a528c339cd4b78fb10b3ef017f657ec17bd", @@ -368,6 +369,7 @@ "postcss": "^8.4.31", "postcss-loader": "^7.3.3", "prop-types": "^15.7.2", + "pyodide": "^0.29.0", "qrcode": "^1.4.4", "react": "^18.3.1", "react-bootstrap": "^2.10.10", diff --git a/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts b/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts new file mode 100644 index 0000000000..d56a804bc6 --- /dev/null +++ b/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts @@ -0,0 +1,156 @@ +import { expect } from 'chai' +import { PyodideWorkerClient } from '@/features/ide-react/components/editor/python/pyodide-worker-client' + +type WorkerMessageListener = (event: MessageEvent) => void + +const BASE_ASSET_PATH = 'https://assets.example.test/' + +class WorkerMock { + static instances: WorkerMock[] = [] + + readonly postedMessages: any[] = [] + terminated = false + private messageListeners: WorkerMessageListener[] = [] + + constructor() { + WorkerMock.instances.push(this) + } + + addEventListener(type: string, listener: WorkerMessageListener) { + if (type === 'message') { + this.messageListeners.push(listener) + } + } + + postMessage(message: unknown) { + this.postedMessages.push(message) + } + + terminate() { + this.terminated = true + } + + emitMessage(message: unknown) { + for (const listener of this.messageListeners) { + listener({ data: message } as MessageEvent) + } + } +} + +describe('PyodideWorkerClient', function () { + let originalWorker: typeof Worker | undefined + + beforeEach(function () { + originalWorker = window.Worker + // @ts-ignore - allow mocking Worker + window.Worker = WorkerMock + WorkerMock.instances.length = 0 + }) + + afterEach(function () { + if (originalWorker) { + window.Worker = originalWorker + } + }) + + it('queues runCode until the worker reports listening', function () { + const client = new PyodideWorkerClient({ baseAssetPath: BASE_ASSET_PATH }) + const worker = WorkerMock.instances[0] + + client.runCode('print("ok")', { + requestId: 'main.py', + files: [{ relativePath: 'main.py', content: 'print("ok")' }], + }) + expect(worker.postedMessages).to.have.length(0) + + worker.emitMessage({ type: 'listening' }) + expect(worker.postedMessages.map(message => message.type)).to.deep.equal([ + 'init', + 'run-code', + ]) + + const runRequest = worker.postedMessages.find( + message => message.type === 'run-code' + ) + expect(runRequest).to.include({ + type: 'run-code', + id: 'main.py', + code: 'print("ok")', + }) + expect(runRequest.files).to.deep.equal([ + { relativePath: 'main.py', content: 'print("ok")' }, + ]) + }) + + it('sends runCode as fire-and-forget', function () { + const client = new PyodideWorkerClient({ baseAssetPath: BASE_ASSET_PATH }) + const worker = WorkerMock.instances[0] + worker.emitMessage({ type: 'listening' }) + + client.runCode('raise RuntimeError("boom")', { + requestId: 'boom.py', + files: [], + }) + const runRequest = worker.postedMessages.find( + message => message.type === 'run-code' + ) + expect(runRequest).to.include({ type: 'run-code', id: 'boom.py' }) + }) + + it('emits run-finished lifecycle event from run-code-result', function () { + const client = new PyodideWorkerClient({ baseAssetPath: BASE_ASSET_PATH }) + const worker = WorkerMock.instances[0] + const lifecycleEvents: Array<{ type: string; requestId?: string }> = [] + + client.setLifecycleCallback(event => { + lifecycleEvents.push(event) + }) + worker.emitMessage({ type: 'listening' }) + + client.runCode('print("ok")', { requestId: 'main.py', files: [] }) + worker.emitMessage({ + type: 'run-code-result', + id: 'main.py', + }) + + expect(lifecycleEvents).to.deep.include({ + type: 'run-finished', + requestId: 'main.py', + }) + }) + + it('reports lifecycle failure and rejects future run requests when loading fails', function () { + const client = new PyodideWorkerClient({ baseAssetPath: BASE_ASSET_PATH }) + const worker = WorkerMock.instances[0] + const lifecycleEvents: Array<{ type: string; error?: string }> = [] + + client.setLifecycleCallback(event => { + lifecycleEvents.push(event) + }) + + worker.emitMessage({ + type: 'loading-failed', + error: 'runtime unavailable', + }) + + expect(lifecycleEvents).to.deep.equal([ + { type: 'loading-failed', error: 'runtime unavailable' }, + ]) + expect(() => + client.runCode('print("ok")', { requestId: 'main.py', files: [] }) + ).to.throw('runtime unavailable') + }) + + it('terminates the worker even when destroy is called after loading failure', function () { + const client = new PyodideWorkerClient({ baseAssetPath: BASE_ASSET_PATH }) + const worker = WorkerMock.instances[0] + + worker.emitMessage({ + type: 'loading-failed', + error: 'runtime unavailable', + }) + client.destroy() + + expect(worker.terminated).to.equal(true) + }) +}) diff --git a/services/web/webpack.config.js b/services/web/webpack.config.js index 35c4e37764..6624a54f4a 100644 --- a/services/web/webpack.config.js +++ b/services/web/webpack.config.js @@ -62,6 +62,7 @@ function getModuleDirectory(moduleName) { const mathjaxDir = getModuleDirectory('mathjax') const pdfjsDir = getModuleDirectory('pdfjs-dist') const dictionariesDir = getModuleDirectory('@overleaf/dictionaries') +const pyodideDir = getModuleDirectory('pyodide') const vendorDir = path.join(__dirname, 'frontend/js/vendor') @@ -407,6 +408,37 @@ module.exports = { toType: 'dir', context: `${dictionariesDir}/dictionaries`, }, + // Copy Pyodide runtime assets from npm package for local serving. + { + from: 'pyodide.mjs', + to: 'js/libs/pyodide', + toType: 'dir', + context: pyodideDir, + }, + { + from: 'pyodide.asm.js', + to: 'js/libs/pyodide', + toType: 'dir', + context: pyodideDir, + }, + { + from: 'pyodide.asm.wasm', + to: 'js/libs/pyodide', + toType: 'dir', + context: pyodideDir, + }, + { + from: 'python_stdlib.zip', + to: 'js/libs/pyodide', + toType: 'dir', + context: pyodideDir, + }, + { + from: 'pyodide-lock.json', + to: 'js/libs/pyodide', + toType: 'dir', + context: pyodideDir, + }, // Copy CMap files (used to provide support for non-Latin characters), // wasm, ICC profiles, fonts and images from pdfjs-dist package to build output. {