diff --git a/services/web/cypress/fixtures/pyodide-packages/tomli-2.2.1-py3-none-any.whl b/services/web/cypress/fixtures/pyodide-packages/tomli-2.2.1-py3-none-any.whl new file mode 100644 index 0000000000..d42f9419ef Binary files /dev/null and b/services/web/cypress/fixtures/pyodide-packages/tomli-2.2.1-py3-none-any.whl differ diff --git a/services/web/cypress/support/webpack.cypress.ts b/services/web/cypress/support/webpack.cypress.ts index 77ed1105e1..84a244c5fc 100644 --- a/services/web/cypress/support/webpack.cypress.ts +++ b/services/web/cypress/support/webpack.cypress.ts @@ -10,10 +10,17 @@ const buildConfig = () => { workerPublicPath: '/__cypress/src/', }, devServer: { - static: { - directory: path.join(__dirname, '../../public'), - watch: false, - }, + static: [ + { + directory: path.join(__dirname, '../../public'), + watch: false, + }, + { + directory: path.join(__dirname, '../fixtures/pyodide-packages'), + publicPath: '/pyodide-packages/', + watch: false, + }, + ], port: 3200, }, stats: 'none', diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts index c79caeaf8d..c59720a7c1 100644 --- a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts +++ b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts @@ -30,6 +30,7 @@ export type LifecycleCallback = ( export class PyodideWorkerClient { private worker: Worker private baseAssetPath: string + private packageBaseUrl: string | undefined private createWorker: () => Worker private listening = false private destroyed = false @@ -40,11 +41,13 @@ export class PyodideWorkerClient { constructor(options: { baseAssetPath: string + packageBaseUrl?: string createWorker: () => Worker onOutput?: OutputCallback onLifecycle?: LifecycleCallback }) { this.baseAssetPath = options.baseAssetPath + this.packageBaseUrl = options.packageBaseUrl this.createWorker = options.createWorker this.outputCallback = options.onOutput ?? null this.lifecycleCallback = options.onLifecycle ?? null @@ -54,6 +57,7 @@ export class PyodideWorkerClient { this.queueMessage({ type: 'init', baseAssetPath: this.baseAssetPath, + packageBaseUrl: this.packageBaseUrl, }) } @@ -97,6 +101,7 @@ export class PyodideWorkerClient { this.queueMessage({ type: 'init', baseAssetPath: this.baseAssetPath, + packageBaseUrl: this.packageBaseUrl, }) } diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts index 8b5f08ad0b..78b621bc2f 100644 --- a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts +++ b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts @@ -15,6 +15,7 @@ export type OutputFileData = { export type InitRequest = { type: 'init' baseAssetPath: string + packageBaseUrl?: string } export type RunCodeRequest = { diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts index 55a4d61ac3..fc4f3c020f 100644 --- a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts +++ b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts @@ -3,6 +3,7 @@ import path from 'path-browserify' import type { PyodideInterface } from 'pyodide' import type { OutputFileData, + InitRequest, ProjectFileData, PyodideWorkerRequest, RunCodeRequest, @@ -14,6 +15,7 @@ type PyodideModule = typeof import('pyodide') const PROJECT_FS_ROOT = '/project' const PROJECT_FS_PREFIX = `${PROJECT_FS_ROOT}/` const PYODIDE_INDEX_PATH = 'js/libs/pyodide/' +const PYODIDE_CDN_URL = 'https://cdn.jsdelivr.net/pyodide/v' function ensureDirectoryExists(fs: PyodideFS, filePath: string) { const directory = path.dirname(filePath) @@ -49,6 +51,7 @@ function syncProjectFiles(fs: PyodideFS, files: ProjectFileData[]) { } let pyodideModule: PyodideModule | null = null +let packageBaseUrlOverride: string | undefined async function loadPyodideModule(pyodideIndexUrl: string) { const runtimeModuleUrl = `${pyodideIndexUrl}pyodide.mjs` @@ -66,12 +69,14 @@ async function loadPyodideModule(pyodideIndexUrl: string) { } } -async function handleInit(msg: { baseAssetPath: string }) { +async function handleInit(msg: InitRequest) { const pyodideIndexUrl = new URL( PYODIDE_INDEX_PATH, msg.baseAssetPath ).toString() + packageBaseUrlOverride = msg.packageBaseUrl + try { pyodideModule = await loadPyodideModule(pyodideIndexUrl) self.postMessage({ type: 'loaded' }) @@ -109,6 +114,9 @@ async function handleRunCode(msg: RunCodeRequest) { const instance = await pyodideModule.loadPyodide({ env: { MPLBACKEND: 'Agg' }, + packageBaseUrl: + packageBaseUrlOverride ?? + `${PYODIDE_CDN_URL}${pyodideModule.version}/full/`, }) const writtenPaths = new Set() @@ -157,6 +165,7 @@ async function handleRunCode(msg: RunCodeRequest) { return originalWrite.call(fs, ...args) }) as PyodideFS['write'] + await instance.loadPackagesFromImports(msg.code) const result = await instance.runPythonAsync(msg.code) if (result !== undefined) { self.postMessage({ diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/python-runner.ts b/services/web/frontend/js/features/ide-react/components/editor/python/python-runner.ts index 4aa9a12025..693a284afa 100644 --- a/services/web/frontend/js/features/ide-react/components/editor/python/python-runner.ts +++ b/services/web/frontend/js/features/ide-react/components/editor/python/python-runner.ts @@ -43,6 +43,7 @@ export class PythonRunner { readonly fileId: string private client: PyodideWorkerClient | null = null private readonly baseAssetPath: string + private readonly packageBaseUrl: string | undefined private readonly createWorker: () => Worker private readonly getExecutionContext: () => Promise private listeners = new Set() @@ -54,10 +55,12 @@ export class PythonRunner { fileId: string, baseAssetPath: string, getExecutionContext: () => Promise, - createWorker: () => Worker + createWorker: () => Worker, + packageBaseUrl?: string ) { this.fileId = fileId this.baseAssetPath = baseAssetPath + this.packageBaseUrl = packageBaseUrl this.createWorker = createWorker this.getExecutionContext = getExecutionContext } @@ -99,6 +102,7 @@ export class PythonRunner { this.client = new PyodideWorkerClient({ baseAssetPath: this.baseAssetPath, + packageBaseUrl: this.packageBaseUrl, createWorker: this.createWorker, onLifecycle: event => { switch (event.type) { diff --git a/services/web/frontend/js/features/ide-react/context/python-execution-context.tsx b/services/web/frontend/js/features/ide-react/context/python-execution-context.tsx index 417bbd79cd..fc74829cd2 100644 --- a/services/web/frontend/js/features/ide-react/context/python-execution-context.tsx +++ b/services/web/frontend/js/features/ide-react/context/python-execution-context.tsx @@ -37,9 +37,9 @@ export const PythonExecutionContext = createContext< PythonExecutionContextValue | undefined >(undefined) -export const PythonExecutionProvider: FC = ({ - children, -}) => { +export const PythonExecutionProvider: FC< + PropsWithChildren<{ packageBaseUrl?: string }> +> = ({ children, packageBaseUrl }) => { const { openDocs } = useEditorManagerContext() const { projectSnapshot } = useProjectContext() const { pathInFolder } = useFileTreePathContext() @@ -99,13 +99,14 @@ export const PythonExecutionProvider: FC = ({ fileId, baseAssetPathRef.current, () => getExecutionContext(fileId), - createPyodideWorker + createPyodideWorker, + packageBaseUrl ) runner.init() runnersRef.current.set(fileId, runner) return runner }, - [getExecutionContext] + [getExecutionContext, packageBaseUrl] ) useEffect(() => { diff --git a/services/web/test/frontend/features/ide-react/components/python-output-pane.spec.tsx b/services/web/test/frontend/features/ide-react/components/python-output-pane.spec.tsx index 3bf2b3bd2e..655dad4ad0 100644 --- a/services/web/test/frontend/features/ide-react/components/python-output-pane.spec.tsx +++ b/services/web/test/frontend/features/ide-react/components/python-output-pane.spec.tsx @@ -298,4 +298,47 @@ describe('', function () { 'ide-redesign-python-output-pane-line-info' ) }) + + it('can load common python data analysis packages on code execution', function () { + const executablePythonFileContents = [ + 'import tomli', + '', + "print(tomli.loads('greeting = \"hello from tomli\"')['greeting'])", + ].join('\n') + + const projectFiles = { + [pythonExecutableScript.filename]: executablePythonFileContents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: pythonExecutableScript.filename, + }, + }} + providers={{ FileTreePathProvider, ProjectProvider }} + > + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + cy.findByText("ModuleNotFoundError: No module named 'tomli'").should( + 'not.exist' + ) + cy.findByText('hello from tomli').should('exist') + }) }) diff --git a/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts b/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts index be6b684d5d..175b0b5027 100644 --- a/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts +++ b/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts @@ -346,7 +346,11 @@ describe('PyodideWorkerClient', function () { newWorker.emitMessage({ type: 'listening' }) expect(newWorker.postedMessages).to.deep.equal([ - { type: 'init', baseAssetPath: BASE_ASSET_PATH }, + { + type: 'init', + baseAssetPath: BASE_ASSET_PATH, + packageBaseUrl: undefined, + }, ]) })