From 9e677a2c1e1be2b7f47540832c45771e2bf71029 Mon Sep 17 00:00:00 2001 From: Domagoj Kriskovic Date: Wed, 29 Apr 2026 12:32:50 +0200 Subject: [PATCH] Use overleaf CDN for loading pyodide packages GitOrigin-RevId: e17ff3387166421a546a9519786d77ba12cdffc4 --- services/web/Dockerfile | 5 + services/web/Makefile | 1 + .../tomli-2.2.1-py3-none-any.whl | Bin 14257 -> 0 bytes .../web/cypress/support/webpack.cypress.ts | 4 +- .../editor/python/pyodide-worker-client.ts | 5 - .../editor/python/pyodide-worker-messages.ts | 1 - .../editor/python/pyodide.worker.ts | 16 +-- .../components/editor/python/python-runner.ts | 6 +- .../context/python-execution-context.tsx | 11 +- services/web/package.json | 3 +- .../web/scripts/fetch-pyodide-packages.mjs | 136 ++++++++++++++++++ .../components/python-output-pane.spec.tsx | 45 +++++- .../unit/editor/pyodide-worker-client.spec.ts | 1 - services/web/webpack.config.js | 4 +- yarn.lock | 4 +- 15 files changed, 203 insertions(+), 39 deletions(-) delete mode 100644 services/web/cypress/fixtures/pyodide-packages/tomli-2.2.1-py3-none-any.whl create mode 100644 services/web/scripts/fetch-pyodide-packages.mjs diff --git a/services/web/Dockerfile b/services/web/Dockerfile index cfd4469e00..9046917eb3 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -27,6 +27,11 @@ RUN update-ca-certificates # the deps image is used for caching yarn workspaces focus FROM base AS deps-prod +# Pyodide wheel bundle (~370 MB). Version + SHA-256 are pinned in the fetch +# script; keep that in sync with the pyodide dep in services/web/package.json. +COPY services/web/scripts/fetch-pyodide-packages.mjs /overleaf/services/web/scripts/fetch-pyodide-packages.mjs +RUN cd /overleaf/services/web && node scripts/fetch-pyodide-packages.mjs + COPY package.json yarn.lock .yarnrc.yml /overleaf/ COPY libraries/access-token-encryptor/package.json /overleaf/libraries/access-token-encryptor/package.json COPY libraries/eslint-plugin/package.json /overleaf/libraries/eslint-plugin/package.json diff --git a/services/web/Makefile b/services/web/Makefile index 4ec73389fa..bae608ede7 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -649,6 +649,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ $(MONOREPO)/libraries/stream-utils/package.json \ $(MONOREPO)/libraries/validation-tools/package.json \ $(MONOREPO)/services/web/package.json \ + $(MONOREPO)/services/web/scripts/fetch-pyodide-packages.mjs \ $(MONOREPO)/patches/* \ | sha256sum | cut -d '-' -f1) diff --git a/services/web/cypress/fixtures/pyodide-packages/tomli-2.2.1-py3-none-any.whl b/services/web/cypress/fixtures/pyodide-packages/tomli-2.2.1-py3-none-any.whl deleted file mode 100644 index d42f9419ef77c9964d6b6624d0fe77284b5cbc5e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14257 zcmZ{L1#sO!vh6i9Gcz+oY{$&Z%*>A2F0ntY7})N$%)~jyPA^cft8KdtuXq+Ps_kTel|vj>@`eXu`yTOglqF^H8yR( zS_C0;uxs6z9f8H#w!*1GA-_KAc}#S}6x~Z-ZNW}Q?Rg+$YqbsMHw0n-x?0?5&{R`y z;!548z-V_kl-(rQO+GWMh5#}#O)M6y6)F0>d}1uALfqgBN%AeWa1m^A?O8rnOdzUq z$sWACpBGjWL0?(5XX@=~RrC4vv|j+vay{ZEk>Y=hOh6M@IU+6q@MHo2Apd)09F4v? zn|=E;Fq8Z?Dtkl8TQ3*{H@$Ag>^&3bolXp;q&?(S01*V!1ig^Ck#Bf(Bk^RTV2I;> zTh8)d=NTGp`(>5S$}(8gpHA+4UCXaH+dH^Fe2&UBpL%*+uGPN(J&fINPX{UX^&LN>T`s#oEVGJ)9o^ebeV9*tjnio zxbtYBuFoXcJj|RnB=NJciML?G1Wyoa9q~l2@!*`kjSsd;Pcp)>up${2)r!9k^S$X; zDu6_k$qLO6av1G~)1>HJ@MC~uY1f_d%stEuzT<&!8>tUHz~Gw^*_J?f^2sJB8W;GE zRcth8akLW^Q;m6NUHHbVUE4X(EszO59I8R3(O;j?^}2KaKD$%;F#b|) z@iogE*>9?kH679nGpav6sZk zZ!rscgia5O1!3VFOVzH~w~g*ff@Jc169{yUKgxmLc(cNn)6%VZ>b`#kz}*Kw z&ngrD9LUFS*#;}gw@Qwq76_1kYwnmwwp4C@3J->`91!8tnJ^5OZV;{`r)-F_OTM1Q zXU7d0HnfD<9WxIr#s{?tEv4xe7=_3JfrG2jok8VH4;wUjiKCf(q9J7Nl#e$`L=LTCt|dxK=g&`xVxfdey^>;0Puxz zvyiGoB=Zqw^-Dt{mI6ksG#t$JSAb6gWulGG?W*v6V{K-vwsmM zM6}9>@EthP)`-!&1@T3o_1Wo{B`XOjY!`Nxe^5)nS$Tv0UTKL7W zY77+W=@`vm_i@MtC`Q=uHsQe*9n^w+?C_YF{f?Ee(I~?hBU-Rp)AhnpktE^Xf|os- zI7W`xh}l80=@C06a#ob)iqC;T#7BdCGG^NT04wtScxz&By@|QU+yN`$bOmCC=wB`H zz$*1DAXP%wN8~Y-2sqpj{CQg`9hafQqyZiDKspmpP&N?$>HxV5f*%Y!^?lAkIr7HxNF?CA~|ven2Q^B+m3#=t3UP&R3fX%B1plG!@#O0dR=DW z)dQYk4BgdrMouA5NW9$GKH}p=G3TeCaZMBJ4d;=WuVO(C$-tol-feU+iU1K|_!0~f z_7kW_fx4VWlFV&mV|-jhEvK_-#Y{I05^3y1zHMBDYR8(F_C8Mpk_BI13e_Tl41S>@|3g{rMIMSGJnfBz0U+da^=tlX zT+n%T939lE-C1^{g!&uhsbKPsN|aw4>X=4Q^S$ag2mup7~CAjk3y{nGs@id zQQ1#C=Lvt2iXtd2xL%=QBJshqs2FjS53oz}s8cPCyo$9YjJt-5edh}}Z=*{-XH$y9 z&bwT#yVRJwu%mtocfSR=7)OM_gwFH{wC+ri6(Ldw5vHDIM>0$G;~)Wx<=az(-!JB# zYy%Pms=bFSSV3F>%lXoz*@e7Tkv*Uk>?iN(@=_z2un-jl7y7EvALL>_V3&#DEI?{<8kTZ@5*ABIT( z9N6Y;OWmEQo5uI+jmK&XPMx)>fRV4~SGw$i#Cv3v9$1V!lFB_O;4ppMyIh8x&NLmvW}eRLDY+1(z)w z&t?@jWGM%f{6cRYiu#G_Sok-L8`Exg50fM9YFl*266isW^sS^5>K$Ni3Xrtg`|W#x zIphK?Qlk$;M~c)LUXD$>c*^~2W0OCZtXM#GCs`0iW5_|cz6_Mc6B1>}uF?|EXSADs z$X&1S)IrgQzRCNLS%v)vd*l}aLKR;FGNC8>J}Id+9mb+2RkKx%)D@J|2X1;1B3H3y zH)_pO`uLAN#_upnPV#cFr%5^4)cRIOTB$1&TB%zYioiM#PMsMy!2`2eLW;HlNRXzP zXuH5H$C4|7j?e6byE3g78?YjNuz+9e^uz(+*%L%FRKv@-Sdh#`xZI%h#B1_(_sK`7 z(n*dHK9umzn;eyXd$q=C$vc5Pd;3q3zjkJ?Bis;aT^{i#abz!Os6?-f$Td=Lp!WG7 z#oNNRYRGP%Rq<%vBho=QK3v1!{FWNBAq#&a$oDVWR11^h$25wdHX=Y<5I1cj&-n9V z*^Ow)yaLMCI#t-d578oQEeW%_aj*l%R0$Hbtfj$XDf#8D)O-%1u86BSfc0=E(5!ju z`y-(3zd#CfIefWi2K}Yz7l3s%La>If4~7<3L*jVn@ACKI%D_37iN>ayTKo@ZA9v@A z@Mzhqp+2b7F&@@E+{JUWE=aq1cZUSlB*t`iFwU{M0@5~GMMQUZ1BX7)3CNyp9^s)Y z|8rS#m;vNhlvxWZVkoQS_G$&KX^TpVf^4e z=GEi7w}QZVALrQ*JPqNR5p_IstZfq}xhidp5f10b(X>UAHZ_lyi zi0(I!yfiVO*nzyJIFwf#fuBeVy||I!liQB~D-MPNtb36S_H%Wh?#~;_(|u#az_zT=z_qb8KPYX%x7U#a9Fm zPL!?v%<1mhi({iA%?l*^Ir9;HpozoJe;k~S0bL>ivDhF?K3}~doEtQ(b zSmDIud9o3bHN#iNA*UM&%t%_o1D;qF2d_%d6hhUzCcVq@8b1k0^l@KsYej4$ocEGN zmXX=@BxzEohK6=)@%j7j!J=zx>Re~{O`934Uo6a-#Eo?|%pa8YjWQ~#-O$NRkTOGa zUz3aQm%}w3f@!1>F+jje{L9|AHxH0(eOp(zm|3xBONRg?OK$6nPc;*4n*2u{;ufRK z=(z5R5`Agx8u*SJJGj2gI{t7dUCF7$v_4+oBtn!$(!H)atL=|XocZm=LrU;sO-f6c zT}$6B1fCTKap$-E?z@3XK8v8Npu(i z7mV9PwtLD8jSRK2?cH11AjY%HVmp4Ii@^}~bWZQZ*T*V7trz4E(&cG{(vvVnzO`h! zb>b<&8#EUcdM0UHuDdv%!Q9!dFvE+f$knC&aBJZWb|u5?0^v43t6RwVAl2LCa(N+A zKRmLgeq@rlv^9;N*6F zmAfgoPgehB2ibq^-yjWvleGz=98Yn$y}NwRH(hiShUgxdf^fQe?T(X5m-6XnWMp7# zEQcp!M)me0-sFI*Ww$mtCAi_X$kkk@gG4{iDr+X_(<}Px$Np;zh)rrXNRvXaUaYq~ zq8kMNv^7#lK4}SM4|A*8!%q+Psp+Zn3?#gdFHA%#|Y(nj6p3J~nsX#HY6OFGuZetJ13kCk>J!mSaCr zvT1Ov-KbEKs_UZQHCkLP>>iZ1q`N2z+)>Jj1D#}V^mZ&7)GX5k&WwQ!?R1?1B9+Vq zX^j1gaBL+;qa?9nbCv2}84jylGN)t{ipuLuv^P3?PttU@lI<|7!FZT4~w?)!J#Kx~^Cu;})ctW)+i6=3k~ zJni#J2PK_}*YH_`EYvPo6jd)Z{q6{e*<>2|`fx=5z%hhz88tq-^rAqSY1n4x^^mk- zX@{6nLaYEeiB$M7vg?YOvGC1Z`7j=QF{7?kV?c>2((=QkdOurZ<8Rwp`*l(H=pS?CGc6Ze`6dSBwN{wltQkc%2SrtZD-kqP1O0hH9OZ_~ve!!CZkI zr8nimiGf@SHfkG!C94+p3$IL7`<2Vj9r*&c{)4ZCI_>FQ9n&G%;`(IebP2(i`qh;4 z>r&O(>^2N->Zu!WuuP+4kGnP)Z$GeH&gG^}^yv^gUu`TV z2ekx9DXIK;<0@KO*T|A%5)Sr;0L(wlzCFjyzKJ}0bv<)EWKAIAxqe1J(OQh~l~Nd;bD!K6kO=08!27A@95vC8{Ve1P8~PM~zjT#KUk99C5zs;~Hxs1>E#_ zX|bduDEb|+$d>&5mrda98C%ZpwOK(HCq2*3!AAX+vTaWLH9JaV=X#UiNa%)OfpT6a zEb_67mQ&Q7Q(0BnE>3)O0a*|2pE9m_0r@rT_OE0-21{wnmnr$*EkSQvy~4M;2Ee~1 z8M7R<9>YaQ*$n2vil@%ZZ6%tMq%49D+;)`cmrp-_h?JLd(_qriVK2e@_e!OXuZm6i zUWzBpXN_5mX{?JpmTLxwtKphja!-n5>zfqJ)E4N{ZDs3yo^lJqu;zRK=N#W>0)_G}io4igvZ(-!cbUC0@Hl3mEZlCov9bPIf^{l1woS-s#+%kBH5Z7I+*Dnx}?=jizXp z^cZ3`vGddte$)EB>2{%k$rr&%_YGO6b~f(t2|-)h3~v&@R~L6mjI(#dMS%w`<$H8) zc;%DY^NKu)?>+b_O1OjP!A{?;SSB%H3Nm*rsxB4EH%xFTJv7GWxB$94M$B25fZ`H^ z(D0{_M7XwK<4mO>n!Yq8sP;=~3dYI1Yd1)5X*O|K|GTGqHd;p5O8_!~$EZpf|~A}Ki$)d4!33IkkPaZgx_cP%F= zhcbL0B6487>u{gJA?gTU*SfrL6lT6j4lQV&w^cTiG(j%8e6mnl+iPjCabbGd>Wr*5 zc}{r^dvvm_Am}HPsjIpxLb*(`YZf_WH!5^hN1tg-&I?q%NpX@sHysQ}9?ASC)DC98&uPP9EwiNe7c`p03_6 zcSeCuyGBd+D(&(T@vb~jw7oa5KW1YRxQEdeRQQZGZKFCD8Cab0ANKZKfIiRc4Xf7= z1q&VLV}Mz<#Vk`e4TVcNJ1bmqx~q}G?u5@vV#MbsTHVbSN1@bs9vj=rm3*`p1D$#V zzg&}-D)$2l;|;yX?B~;B{BXYxpZ3pp9WRoiw?~8Otqv>r<+Zi%V4o`E)tzb+6U>99 zcJIYK5KIeV2h+F6pG*jXzfdUOOnaUO93ebYs^0zW*U+d!gZ;wEI*{0l?HI3@$ru@D zv7Qos*?r_m7s#U9T8l3&Cm@VHwbQ(*ykzlC!{m&e02ymDW)SE|O@6t@zs;Saz;yH* z%mmuWjKEvH#qaCXC?xg2li!!0x`v(HfBq!40To`>@Mgu2YJ+`A*bmgF3|?nViH*R; zfAmGXB>8}YFPSI6xJ%|<*DND9v4nlM)?HG<;(o`v!;sdJno?v`H5_pMuF%a{Z}&8p z+V%}eWm#cxqsl_LNn0WP$_cMrdkJoVX6yCqf98(thrx7izySagcmUwf@vp4fH?u!^ zqe68%hc!;*k17Kv$1cfI@)7x^qP}%{If4qoUy$J%@OSWHaWom_vbgAK166lh?&M;f z+$SMMAw~(q?Ry)}NHKiCwX>&gC?YFyO(`G}Tp#%)4GN2{Pb#m3`Aam4VRaUiJoIc$reqliEx7|^qdE4WX!XAq!&Tx8y|Ckx}Ac(C3w zN|!7(6*>1xBbsDt)kG@KfIMJ1b`ernsP@G|hlr6zCXzRcSqNgTrBz-`iV-eL8cHcO zXdL+yG%WAg6kS5;S=`Tyl`74*C7qlrVc*RQ4ey8!;SghzFBt#z9rKEpyF^vv#^4LzL|Q%dx( zh0k97T!}RF4N0H%D05?Cwt2nOiK61Vf)JqJ&2?yCS3u5JdtzO5rxq5 z)CUgNuvRSoV(El>_i9S~Rf-3JXH#_m&hEjV9mdv}(sPks1LzXgPt#9PxQ>7R$wC zh6bY>4K?YfB&0ygZBN>HHxx-F)G%8_D9L7;5j|Z4Pf3>~0{eW27BNzHhVNN)V{C)m z>&3NP;oA?{&3wm@0qv`F+!dzkHYw!l7MM74~BtfE5pZo}7)+0f=9@nac08RolB0(E@`eE&8!x8sslS1OC zukK;ilgsZI!fs<(A4sV!L%ys#d%qL=TJm|pq!SlH4PVm&(+RzJL6;2S;b_CUzM;QJ z6c;NU|DNkms43(xu^fgaRA}yODr*9=9H%!vltUoM%&T&V=-{caHvagIh*4X?7&Xl7 z!NS}C7qStdxtA&7)USelJ|_;A^sxNc5lU}Ux}#XNlRA*IqK z5;x7K%eCL}INmtLYDTY#OHC?bI=ivgS?06L%6*CoFzjBp9~de8pu+71Z&QJTiz(U8 zMd2(QH_ofcKzCx4Xmr*{!l9Sgg|YmR2l-_RgIxgJ4@Og{d%(Y{M)3w2e@gsunXirEvBOZJmded^v}Np0L1^b z*wK^mA2yq+s9(#`kI6GAOO8)SsY#AADMLFVC^B=f?SlgTlSfa}73dBM05Jdgun_!< zhk=!mm63(f)XLd~!OGs;fk{?ML`+^qOiy{#evKKq^NtRZhr&O$iRV#V#zj;WkD9kC zrcPAH)@&BJ6!+zAYD9hxotfA%xjEXol%&2Ar_+eu?PzI~@D1>E0mkR!^9tF9GigZj z)99|xy=!liCghYR>HlQDX3u*1z^!mI}#I*RQGW%*{#n+^>OH1`g!Pvc{3@v%I& z942F*A;sz+ogs^I4|TMn4D2Uh-)vvx41@YznH=x#b}tfw-cHN8$3J;{i%M3Z;6h%$KUnQ@V55oFnOoqrY&Mp)zU_Cj71b`BZxGdcv~Ie)fE4Oe%z#W0 zX>>MvZ^46|Pcks?4|-(yoy&?|fe~(Uf@N+rT)#Wqp}eSoxVA3pRDqA+(yDchh>|-F z+jwt@)g57Yp|@ zPLqkwX#*$i5Ry_t(_~c5+)b!9rF>vCwkd_{(@lZpR@Jq_abT^=SuH86fS99#))_Ib zj1}9@n~52T;6P@2_UAp&5?76KQ0yf2K~7naA3ahP(KZk7*_F;=?HU_Mf|VP@N$~`S znSQC)Y?2mLiW;gqHOzpeMIjnCs4&xtjc%cD=(7l7O!76@7}FN*NvfcUZKS;mQ_5^n zVkBS*6i;A$+at{!fqHsE`fk`Q!4B)BphCfAv^7 zF;yW^AyuIR?KAhaCXCOiZcvqE6*`pC(7hDQLf7=Bo6-c2=rwzZ@o-|)_?lR7ky!P_ zrnj}HcEEE@<-F6}<+!&ODQXMT`V+*PEIQvV#FHOtYcefCy;98?|4}%0y)Z@Xy*6IH z;c+K@LeNnL%NO(7sBXg}rmUZJQAgVQn@`5|>YfD*W$wC|bV+ZdJFJBD&RG$fb;So6 zT&x&0-&!=OSNIZQr?j16D+&zH#GF7Lm7Z$de z{WDWzEk#k^l=LWAHs$<49E+5thR^%v)|?vWP`X@9i0SJ%`rI*I+!XQB*W(@4qJwss zZHrdzLp~|!oy6ktc&16I3hiU7C4Bm5uI7O+Qgxc-io{fQb7M|OFUE#KSUcm9o44yYvp`)d_MfzJKFMbe%);+ZWX>Rl@<-T+mxKX zEAmSdU^K(8O;HpnHMm$O2SJk3X^TnqtPk5Ee-}O$hQiAlc;|^pCY`|=XUa`x*OjX% z*@&ThJ6KcR&yW)PeR+Aqqtxv%y0d`VfkamTpdXNr^-+HpO%qzuMX{#wH`h=cY(?wKPas8O?f>=C2Xk>8=fhQ@GF1-!N?%3KESUDTSM;ZmP zW2GW2zT`L|ko&QUSNjEG9uG^rjbe;iDgBhaZUf4O=^%{W^aDyG0dOO!1BNfSxu%cG}CfY08YmT=u}0QaNG*M<=mV z3jar1ZvZ2Fgu}KE?IGl9wLjfkIY(q7hju_kg;pP1i%4#nti5{py>#Wfq*g+jSc~5w zT1hsJV!O4fH{~}pJ}b-2YR_`rhJjjou+`y={GG>ayE?fM?VS!(Qb<$bGvCh3jEBo6 ze+(myO_m2;CsFBW*n-fdiag94v+2-;1jq^~r&IB{O4UW*>S#-~=(I?iFZ zVu~>IqQoOaRs(k>$vQ;~Qf`4eGwreXryqw=7eq}3n$=4IYAvH(aV^Idq`&TCz5+~% zl(iO!{b#Qd9pRY^Hf6;3W1R`RNG!}EmU98IZTJT6m=~4NPxeW3S)qw3WxyLa_VmOV zet(bvEZae*pp~^S-~lj{j`vP_BvbaRJeX)a7@8I?4{KrMX`fIBt#hUJ(~Uo~;2Q~N z3eBhqzEp+^xipq3RQ&hoUwsWZgthQi>G-lwe3h%bUDL8uQ6npg`kG;xQi}e#+K}X9 ztWJ|Qbi|<57;s70^QH^DBp7!tDt8tIWSz=px@dL0A?|D~5|eC>Q(K9t>hSE5YSa_v z^*l>-ObLmAi7q_y8WYL1UDaRA6lKC)ffo#}WssZ7mD=8wbY3$^t%+x2$44B3D?zTj z1P;8Dvan0l`ktZ*$qIvIIdKJhg=VZ+;Jm9(O+W#rQiLveuWb~gaq9RIE41-1fWZ`}`?U@CAfFo)aC8k-JD zebpt2V~=jZb5nAs>^xFtq#GdiwO)E9OFQ z)nvHzG$^djMkD>5aX?*7k_m`4mI^rmGRUytRHisX!e+xo)J;ldZf9!j#*88;YFbF- zli@pucaaROaaH9XJ1jDTydkI@kp)GX;a;1%vpiVP6gX9~|G3^R=e{>JAZjTj5QJ58${A%$bZpO=ztSD41;L+9h6AgHR_zC z(}n{qSw8u#V#;r6OQM4Al^OZ1hy>FmGcBuJc)5TUI)g%3<#?4{1OYc**M7qMjOrVx zYB_Tvr0K3)l5N;+rfu%{fHj(wu?aPCCt2jL>k7N~vPgfG%gWaw?F*tUe*NYUlGE)p zwrG=9hFM!S7BO5UKyj7`7aF7;9=sHHev?sDm9Enci?vae)@ypQ)`&_Sq2A(kO?oW1Z3JobBx+^dJgP zu;lG`UBZJ+Pk|;s2WmBesdl=f85RZ2X{E9Cnx~)y^%vA74l@Vqf{@3}=3j4G8fMw+ zM?+ouDKNZR4eOKbEao3Y&7d*sCTBb!?VyrZTOC-MR5yJJd%tZ!4V_P<-Dwl?JTgAT z5m^qDteJ=ti)#(F1)P*NJ*7q#@e2(x!1>1_o{R~zF?gVo4HM(wl%A?vefmyE+OD-J zI#n!k^QWQ2k$Q--I^X+jo2W>VE6S5jFl1yo#mjhXHmsQQ2KLWVsGOODAkZ412C8RA z7UA*7%!q9n1+DAvvp4)sPrl5MkGnNg6V5ce{&lZzcd<-PnP5r&o!zJ(Q2F%@uRgD{ zD(W%TbtLa+TU!x-Ss|X<;ShZ%ZOqN19kh$e(wOuY+W`L}LV{5N(VKW)90wXTuY7)_ zp#g{OAP+e;f7A7qi`tzU8yh(~qumlhFI6Wy5#^*RV!kKE!nqTyqlcjApqO-@NvCk2g-_S3v3cn+Y{D+KO0?@^cWGP?vPe9}cEIe_qrU)b$kL62;N`EH$}v*9xQgQA zO|y~X41YPr=G2wwl)yzt<`$d0QPV2TyowknYdLYYJAj^O9~H^fy;EB8;5h0^vBQap z!`XQwLhPY;zP?_b9Sf~SD{TrLI9_F>rEPF+N+L4UM|ja-TMw1)0DJy)Wwi;dgjW^l z+TE14p0M*fkX&iIK$Szu7^S%iJ55*mvaF8Po{@@X%~-M|0!xkN&UEa2ao_k$UHxIt zC4QTNxS}xZ{89$z=&0FR2=YfMe)ne1z z@J5u{VOF~7)u;DfXJ_|ie9K{1&z$U}t76)23C;!{S2OP;WN?(lfO|wERcnFDEgL*t zUXyP@P$exZ!XE3lju~9D^(I*UG2Fx3KI~kp@^knBGO`rAu=Pf)jZBHv7nmZYUmdMh z$7D>987L=v%5sGp26%H5@9(Qc6X^?7d4dEB=LsLa5LBOIMD1Poy$dlPY?)alHSAU= zP!KXWCwiDI^z+3(3w?^NSW@(Ic|@EA+F@T`GWA!JKmB@6z&~0Sr+dL(N<_+2%#gs^` z#r{^ao%xtyeG@J;SwRF3KVT=Vw4D*dIin-uc(U!}h% zC-6^k$UpM=As6rQ9igV6|KmQ7cqeA^e)62WQYW}ogVzeXqWMB+Ys$`es}&0d zZY7~<`AyZwpL8d`5WMkc7rd>DLv>B1O^&A~5in_^F4l2{E5tdM(9P+m%caghW6I{$ z2Cbt_R6B^^qKt|zQR!bZSQ%Z!DB>V85D~?Z2Ew*}e}bd8BKtt`6A@k#i2pXMrGota z+pemOR$BUssB(o^6g<^|JK^K&7( zxFo`!^FBR>Fn@a+jQh9?EB2-Hn)>*`lob~e2^fGi#OlqKhHre&e=Uf!t;Lx(!&S3W z^_09xl^_II)z|NYuRU8}3N+joKdzprWNuhAC>6aYkt72idC3IU!6j~e(cUBSOoLn* z`r;f2CZ}|$o_T=)2__CB7}Ke627hi>NkD)CvP&>j-bZ=d)P6xef7#1+1iXB0v$KDC zN9wda8j6Co>p%d$LqW;zL4f5kJGF`AV?-M!5zvnG=jR6SVMDy-w`!4$;2+iI9}3Vvi);RmVxuf3qM$75Iz9TWOb{SA z>(d1RPppEe)!^?>I#SY-i;gPQ)){2JDgO(Mh3 zR{8?(3dGP2AWq&pOkbpR0TzfLj9trDPaGg|KGuAi5nAr_6eRdaAod=OE=xA{R;K5E?k?VL1CJvt( z&0x46^yFj<&X;lN^;xr@TVEMeW^D~JCI~a#ml213-y1*&?{i=`q(wUWPr3?S6z>l# zR<~CPJR;1%Gj zKK{5H2q-$(e=ni>^P>N``3L;t^nVJf{)YcuXY~KU0Dyl0?jP6rANc=P9sP~|JJa-U zwE7?C{~z=}^G<&w{}y8ZjdUXZU*x}J*}uVmYnuNCn~?kq{4a&`Z|vVX=D)G!|HA&C zIr2ZM=HFC*3rzo}(j)($$M`=~|B{>j=K4Es{F`fz^}libGm898^!Kd)7g0li9@~E- a`u{ONQ3e9?A6LWt*|q<~3?Ytx^!^`9mYD|t diff --git a/services/web/cypress/support/webpack.cypress.ts b/services/web/cypress/support/webpack.cypress.ts index 84a244c5fc..8f78d25676 100644 --- a/services/web/cypress/support/webpack.cypress.ts +++ b/services/web/cypress/support/webpack.cypress.ts @@ -16,8 +16,8 @@ const buildConfig = () => { watch: false, }, { - directory: path.join(__dirname, '../fixtures/pyodide-packages'), - publicPath: '/pyodide-packages/', + directory: path.join(__dirname, '../../public/js/libs/pyodide'), + publicPath: '/__cypress/src/js/libs/pyodide/', watch: false, }, ], diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts index c59720a7c1..c79caeaf8d 100644 --- a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts +++ b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-client.ts @@ -30,7 +30,6 @@ export type LifecycleCallback = ( export class PyodideWorkerClient { private worker: Worker private baseAssetPath: string - private packageBaseUrl: string | undefined private createWorker: () => Worker private listening = false private destroyed = false @@ -41,13 +40,11 @@ export class PyodideWorkerClient { constructor(options: { baseAssetPath: string - packageBaseUrl?: string createWorker: () => Worker onOutput?: OutputCallback onLifecycle?: LifecycleCallback }) { this.baseAssetPath = options.baseAssetPath - this.packageBaseUrl = options.packageBaseUrl this.createWorker = options.createWorker this.outputCallback = options.onOutput ?? null this.lifecycleCallback = options.onLifecycle ?? null @@ -57,7 +54,6 @@ export class PyodideWorkerClient { this.queueMessage({ type: 'init', baseAssetPath: this.baseAssetPath, - packageBaseUrl: this.packageBaseUrl, }) } @@ -101,7 +97,6 @@ export class PyodideWorkerClient { this.queueMessage({ type: 'init', baseAssetPath: this.baseAssetPath, - packageBaseUrl: this.packageBaseUrl, }) } diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts index 78b621bc2f..8b5f08ad0b 100644 --- a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts +++ b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide-worker-messages.ts @@ -15,7 +15,6 @@ export type OutputFileData = { export type InitRequest = { type: 'init' baseAssetPath: string - packageBaseUrl?: string } export type RunCodeRequest = { diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts index fc4f3c020f..7bf4bbfbae 100644 --- a/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts +++ b/services/web/frontend/js/features/ide-react/components/editor/python/pyodide.worker.ts @@ -15,7 +15,6 @@ type PyodideModule = typeof import('pyodide') const PROJECT_FS_ROOT = '/project' const PROJECT_FS_PREFIX = `${PROJECT_FS_ROOT}/` const PYODIDE_INDEX_PATH = 'js/libs/pyodide/' -const PYODIDE_CDN_URL = 'https://cdn.jsdelivr.net/pyodide/v' function ensureDirectoryExists(fs: PyodideFS, filePath: string) { const directory = path.dirname(filePath) @@ -51,7 +50,7 @@ function syncProjectFiles(fs: PyodideFS, files: ProjectFileData[]) { } let pyodideModule: PyodideModule | null = null -let packageBaseUrlOverride: string | undefined +let pyodideIndexUrl: string | undefined async function loadPyodideModule(pyodideIndexUrl: string) { const runtimeModuleUrl = `${pyodideIndexUrl}pyodide.mjs` @@ -70,12 +69,7 @@ async function loadPyodideModule(pyodideIndexUrl: string) { } async function handleInit(msg: InitRequest) { - const pyodideIndexUrl = new URL( - PYODIDE_INDEX_PATH, - msg.baseAssetPath - ).toString() - - packageBaseUrlOverride = msg.packageBaseUrl + pyodideIndexUrl = new URL(PYODIDE_INDEX_PATH, msg.baseAssetPath).toString() try { pyodideModule = await loadPyodideModule(pyodideIndexUrl) @@ -93,7 +87,7 @@ async function handleInit(msg: InitRequest) { async function handleRunCode(msg: RunCodeRequest) { const { fileId, executionId } = msg - if (!pyodideModule) { + if (!pyodideModule || !pyodideIndexUrl) { self.postMessage({ type: 'output-line', stream: 'stderr', @@ -114,9 +108,7 @@ async function handleRunCode(msg: RunCodeRequest) { const instance = await pyodideModule.loadPyodide({ env: { MPLBACKEND: 'Agg' }, - packageBaseUrl: - packageBaseUrlOverride ?? - `${PYODIDE_CDN_URL}${pyodideModule.version}/full/`, + packageBaseUrl: `${pyodideIndexUrl}${pyodideModule.version}/`, }) const writtenPaths = new Set() diff --git a/services/web/frontend/js/features/ide-react/components/editor/python/python-runner.ts b/services/web/frontend/js/features/ide-react/components/editor/python/python-runner.ts index 693a284afa..4aa9a12025 100644 --- a/services/web/frontend/js/features/ide-react/components/editor/python/python-runner.ts +++ b/services/web/frontend/js/features/ide-react/components/editor/python/python-runner.ts @@ -43,7 +43,6 @@ export class PythonRunner { readonly fileId: string private client: PyodideWorkerClient | null = null private readonly baseAssetPath: string - private readonly packageBaseUrl: string | undefined private readonly createWorker: () => Worker private readonly getExecutionContext: () => Promise private listeners = new Set() @@ -55,12 +54,10 @@ export class PythonRunner { fileId: string, baseAssetPath: string, getExecutionContext: () => Promise, - createWorker: () => Worker, - packageBaseUrl?: string + createWorker: () => Worker ) { this.fileId = fileId this.baseAssetPath = baseAssetPath - this.packageBaseUrl = packageBaseUrl this.createWorker = createWorker this.getExecutionContext = getExecutionContext } @@ -102,7 +99,6 @@ export class PythonRunner { this.client = new PyodideWorkerClient({ baseAssetPath: this.baseAssetPath, - packageBaseUrl: this.packageBaseUrl, createWorker: this.createWorker, onLifecycle: event => { switch (event.type) { diff --git a/services/web/frontend/js/features/ide-react/context/python-execution-context.tsx b/services/web/frontend/js/features/ide-react/context/python-execution-context.tsx index fc74829cd2..417bbd79cd 100644 --- a/services/web/frontend/js/features/ide-react/context/python-execution-context.tsx +++ b/services/web/frontend/js/features/ide-react/context/python-execution-context.tsx @@ -37,9 +37,9 @@ export const PythonExecutionContext = createContext< PythonExecutionContextValue | undefined >(undefined) -export const PythonExecutionProvider: FC< - PropsWithChildren<{ packageBaseUrl?: string }> -> = ({ children, packageBaseUrl }) => { +export const PythonExecutionProvider: FC = ({ + children, +}) => { const { openDocs } = useEditorManagerContext() const { projectSnapshot } = useProjectContext() const { pathInFolder } = useFileTreePathContext() @@ -99,14 +99,13 @@ export const PythonExecutionProvider: FC< fileId, baseAssetPathRef.current, () => getExecutionContext(fileId), - createPyodideWorker, - packageBaseUrl + createPyodideWorker ) runner.init() runnersRef.current.set(fileId, runner) return runner }, - [getExecutionContext, packageBaseUrl] + [getExecutionContext] ) useEffect(() => { diff --git a/services/web/package.json b/services/web/package.json index 5e8d3da33c..1cb392da50 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -26,6 +26,7 @@ "nodemon": "node --watch app.mjs --watch-locales", "webpack": "webpack serve --config webpack.config.dev.js", "webpack:production": "webpack --config webpack.config.prod.js", + "pyodide:fetch": "node scripts/fetch-pyodide-packages.mjs", "webpack:profile": "webpack --config webpack.config.prod.js --profile --json > stats.json", "lint": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --max-warnings 0 --format unix --ext .js,.jsx,.mjs,.ts,.tsx .", "lint:fix": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --fix --ext .js,.jsx,.mjs,.ts,.tsx .", @@ -374,7 +375,7 @@ "postcss": "^8.4.31", "postcss-loader": "^7.3.3", "prop-types": "^15.7.2", - "pyodide": "^0.29.0", + "pyodide": "0.29.3", "qrcode": "^1.4.4", "react": "^18.3.1", "react-bootstrap": "^2.10.10", diff --git a/services/web/scripts/fetch-pyodide-packages.mjs b/services/web/scripts/fetch-pyodide-packages.mjs new file mode 100644 index 0000000000..dc10bc856f --- /dev/null +++ b/services/web/scripts/fetch-pyodide-packages.mjs @@ -0,0 +1,136 @@ +/* eslint-disable @overleaf/require-script-runner */ +// This script doesn't work with ScriptRunner because it is run during the build process. +import { createReadStream, createWriteStream } from 'node:fs' +import { mkdir, readdir, rm, stat, writeFile } from 'node:fs/promises' +import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' +import { execFile } from 'node:child_process' +import { createHash } from 'node:crypto' +import { promisify } from 'node:util' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const execFileAsync = promisify(execFile) + +const SERVICE_WEB_DIR = path.resolve(fileURLToPath(import.meta.url), '../..') + +// Pinned pyodide release tarball. Keep PYODIDE_VERSION in sync with the +// "pyodide" entry in services/web/package.json. When bumping, update both +// PYODIDE_VERSION and EXPECTED_SHA256 together; fetch the hash via: +// curl -sL https://api.github.com/repos/pyodide/pyodide/releases/tags/ \ +// | jq -r '.assets[] | select(.name=="pyodide-.tar.bz2") | .digest' +// (strip the "sha256:" prefix). Cross-check by downloading the tarball and +// running `shasum -a 256 pyodide-.tar.bz2`. +const PYODIDE_VERSION = '0.29.3' +const EXPECTED_SHA256 = + '458e8ddbcbb6e21037d3237cd5c5146c451765bc738dfa2249ff34c5140331e4' +const TARGET_DIR = path.join( + SERVICE_WEB_DIR, + 'public/js/libs/pyodide', + PYODIDE_VERSION +) +const TARBALL_NAME = `pyodide-${PYODIDE_VERSION}.tar.bz2` +const RELEASE_URL = `https://github.com/pyodide/pyodide/releases/download/${PYODIDE_VERSION}/${TARBALL_NAME}` +const COMPLETE_MARKER = path.join(TARGET_DIR, '.fetch-complete') + +async function download(url, dest) { + console.log(`Downloading ${url}`) + const res = await fetch(url, { redirect: 'follow' }) + if (!res.ok) { + throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`) + } + await pipeline(Readable.fromWeb(res.body), createWriteStream(dest)) +} + +async function sha256(file) { + const hash = createHash('sha256') + await pipeline(createReadStream(file), hash) + return hash.digest('hex') +} + +// The version subdir only needs what pyodide fetches via packageBaseUrl +// (wheels, their .metadata sidecars, and lib*.zip shared libraries). Skip +// everything else: +// - core runtime (pyodide.mjs / asm / stdlib / lock) lives one level up, +// copied from the npm package by webpack CopyPlugin. +// - *-tests.tar / test-*.zip: per-package test fixtures and pyodide's own +// test packages, not used at runtime. +// - console*.html, python / python.exe / python.bat / python_cli_entry.mjs, +// README.md: REPL UI, CLI shims, and docs. +const TAR_EXCLUDES = [ + 'pyodide.mjs', + 'pyodide.asm.js', + 'pyodide.asm.wasm', + 'python_stdlib.zip', + 'pyodide-lock.json', + '*-tests.tar', + 'test-*.zip', + 'console*.html', + 'python', + 'python.exe', + 'python.bat', + 'python_cli_entry.mjs', + 'README.md', +] + +async function extract(tarball, targetDir) { + console.log(`Extracting ${path.basename(tarball)}`) + // Tarball contains a top-level pyodide/ folder; strip it so contents land + // directly in targetDir. + await execFileAsync('tar', [ + '-xjf', + tarball, + '-C', + targetDir, + '--strip-components=1', + ...TAR_EXCLUDES.map(p => `--exclude=${p}`), + ]) +} + +async function main() { + try { + await stat(COMPLETE_MARKER) + console.log(`Pyodide ${PYODIDE_VERSION} already present at ${TARGET_DIR}`) + return + } catch (err) { + if (err.code !== 'ENOENT') throw err + } + + // A prior run may have left a partial install without the marker; wipe it + // so extraction starts from a clean directory. + await rm(TARGET_DIR, { recursive: true, force: true }) + await mkdir(TARGET_DIR, { recursive: true }) + + const tarballPath = path.join(TARGET_DIR, TARBALL_NAME) + try { + await download(RELEASE_URL, tarballPath) + const actual = await sha256(tarballPath) + if (actual !== EXPECTED_SHA256) { + throw new Error( + `SHA-256 mismatch for ${TARBALL_NAME}: expected ${EXPECTED_SHA256}, got ${actual}` + ) + } + await extract(tarballPath, TARGET_DIR) + await rm(tarballPath, { force: true }) + + const extracted = await readdir(TARGET_DIR) + if (!extracted.some(name => name.endsWith('.whl'))) { + throw new Error( + `Extraction did not produce any wheels under ${TARGET_DIR}` + ) + } + + await writeFile(COMPLETE_MARKER, '') + } catch (err) { + // Leave no partial install behind, so the next run starts clean. + await rm(TARGET_DIR, { recursive: true, force: true }) + throw err + } + + console.log(`Pyodide ${PYODIDE_VERSION} ready at ${TARGET_DIR}`) +} + +main().catch(err => { + console.error(err) + process.exit(1) +}) diff --git a/services/web/test/frontend/features/ide-react/components/python-output-pane.spec.tsx b/services/web/test/frontend/features/ide-react/components/python-output-pane.spec.tsx index 655dad4ad0..1e5d8d12df 100644 --- a/services/web/test/frontend/features/ide-react/components/python-output-pane.spec.tsx +++ b/services/web/test/frontend/features/ide-react/components/python-output-pane.spec.tsx @@ -325,9 +325,48 @@ describe('', function () { }} providers={{ FileTreePathProvider, ProjectProvider }} > - + + + + + ) + + cy.findByRole('button', { name: 'Run Python code' }) + .should('not.be.disabled') + .click() + cy.findByText("ModuleNotFoundError: No module named 'tomli'").should( + 'not.exist' + ) + cy.findByText('hello from tomli').should('exist') + }) + + it('auto-installs python packages imported by the executing script', function () { + const executablePythonFileContents = [ + 'import tomli', + '', + "print(tomli.loads('greeting = \"hello from tomli\"')['greeting'])", + ].join('\n') + + const projectFiles = { + [pythonExecutableScript.filename]: executablePythonFileContents, + } + const ProjectProvider = makeProjectProvider(projectFiles) + + cy.mount( + executablePythonFileContents, + }, + currentDocumentId: pythonExecutableScript.file_id, + openDocName: pythonExecutableScript.filename, + }, + }} + providers={{ FileTreePathProvider, ProjectProvider }} + > + diff --git a/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts b/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts index 175b0b5027..814251e0dc 100644 --- a/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts +++ b/services/web/test/frontend/features/ide-react/unit/editor/pyodide-worker-client.spec.ts @@ -349,7 +349,6 @@ describe('PyodideWorkerClient', function () { { type: 'init', baseAssetPath: BASE_ASSET_PATH, - packageBaseUrl: undefined, }, ]) }) diff --git a/services/web/webpack.config.js b/services/web/webpack.config.js index 7bd7b2c247..75a75f663d 100644 --- a/services/web/webpack.config.js +++ b/services/web/webpack.config.js @@ -411,7 +411,9 @@ module.exports = { toType: 'dir', context: `${dictionariesDir}/dictionaries`, }, - // Copy Pyodide runtime assets from npm package for local serving. + // Copy Pyodide runtime assets from the npm package so the loader is + // always available. Python package wheels are fetched separately by + // scripts/fetch-pyodide-packages.mjs into the same directory on disk. { from: 'pyodide.mjs', to: 'js/libs/pyodide', diff --git a/yarn.lock b/yarn.lock index 99a2f7efaa..87e34425d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7437,7 +7437,7 @@ __metadata: prop-types: "npm:^15.7.2" pug: "npm:^3.0.3" pug-runtime: "npm:^3.0.1" - pyodide: "npm:^0.29.0" + pyodide: "npm:0.29.3" qrcode: "npm:^1.4.4" rate-limiter-flexible: "npm:^2.4.1" react: "npm:^18.3.1" @@ -28272,7 +28272,7 @@ __metadata: languageName: node linkType: hard -"pyodide@npm:^0.29.0": +"pyodide@npm:0.29.3": version: 0.29.3 resolution: "pyodide@npm:0.29.3" dependencies: