From 87b0ae3ece8b3bf1f5adc7848aa33db4d3e95ac4 Mon Sep 17 00:00:00 2001 From: skirui-source Date: Thu, 13 Jul 2023 16:36:59 -0700 Subject: [PATCH 01/15] create pr- tco benchmarking project --- .../amazon-deeplearning-ami.png | Bin 0 -> 44755 bytes .../xgboost-rf-gpu-cpu-benchmark/Dockerfile | 7 + .../xgboost-rf-gpu-cpu-benchmark/hpo.py | 199 +++++++++++++ .../notebook.ipynb | 268 ++++++++++++++++++ 4 files changed, 474 insertions(+) create mode 100644 source/_static/images/examples/xgboost-rf-gpu-cpu-benchmark/amazon-deeplearning-ami.png create mode 100644 source/examples/xgboost-rf-gpu-cpu-benchmark/Dockerfile create mode 100644 source/examples/xgboost-rf-gpu-cpu-benchmark/hpo.py create mode 100644 source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb diff --git a/source/_static/images/examples/xgboost-rf-gpu-cpu-benchmark/amazon-deeplearning-ami.png b/source/_static/images/examples/xgboost-rf-gpu-cpu-benchmark/amazon-deeplearning-ami.png new file mode 100644 index 0000000000000000000000000000000000000000..c9a9c7cea88aa8642d5eb7ba2620209a22b17d26 GIT binary patch literal 44755 zcmbTe1yCGa8z$O70tuQBf=fPvySoN=m!QERcyPA>K_|%I8iKpKySrO(8{B<6-@pIX zR^3}$dvDcHOpP?%ea?B`#||Ni@)Bss_{abNph-!JDgyw5KLEfvy_Z5Rl@g5rKWH_IvKDPh4(&8cpNt`5%t!;@&03KW@c)3 zy=mZVieqMIijI6j_8y+tj)}#XB~g?vPCEOes-XXeSK#9eqaEUi@aBau`1m>d7Fp!& zf1bYe?NI#p*&E{*8BADz-+uD~4~_~|>_49s2mb$lbQ5|xe59*?wD8ms#?DUcwK2f0 z;8KgD$MW}tn$-19nN&qY4)3-cb`(g+F<*ULjx)Hmx{piPMKR)Ay3f9Mjevx-l-)j} zjVf+uBHRnCJgj6Sx6C&mOezR?2C*O^qoMx@Y58KvcNZ9Y_0Z#5*I3U*#P6)kZ2#Pn zZupB)^K%j{h4|2`zehQXAb*q|o0$l0aoz5ULYV6vKDPbA@%Ml`qUv-k8?E=0{1hdvR zkNJD|H!sgZgEQOxyicz&k!7Ss;E_W3o>F;uc#G-idmscK6;4l+zF1jx8JFw1>b8>H z9PEZnGB(Y==b#2J~okLB5c&kv9Ukqt^`wr`mOq-zdmQ1!XdajF&@pnEk32n=T z&9TUl5#d2bI+~-ctJ~YVrtqGR;B4r#^yivDa}5a>$6L_UCgO9fs?WB*957L($u5Ni zTk|-6(U+3KOYc3uaGac&qAM*B@VN4?o#k@j#G$7z?)Vj5dhNNoILOX6rr%;!9?)DI z)WFl2OL?XKaH?VkeAyz9*&Ys2EIRn3FTtqOAl&1UH0X0vQ z%_+`yI@@i3^dT;jbcrXo=Fi232mx3#yi{34Yodv@hE^OpZC*4pH#c>0?$G!QEDs6 z1MS8W0`TFT)LYhE1#~nPe3QAQiMx%_d+9-1;Li?8%b~%w)oL}tLCjKl$AJf42Aee2%PGRo?ML%q+|}U)&^aN|a|cCZ8v%(W0TD z?eFi8>NizfZpfuuJ=0Ov0)RjK!}e&ZHQPX0Vd4GVu=WmvUR=L%C|X%{HR+p^t@mnY zSbW4dxPN?Dx7DTZ9?4O5>Kk}Wb~XoV2aN5`9H$m7SDNf@ziMc30(2>YwdMKTyp*G( zvmSroq_ZnR2c*p?k;qeh|CWRw-%8KS&A<4PBrmI|Dn}>9$&*HTHc?X9IF;4P4fg5O z8cgrC#aEje)*Q^JN9 znmjKT7A^_OeadS$L`juw^aKF9&wM^>P23tK%S#Pjv#E6+wha0_mg03y;`;i>;F?%* z-+3I?u4&}q9Zq|XF2ZHMUzC_%e*GQ1dj}iV<{3U5`@cMCi4}HtVGGy2?T{L)=gV4z zjULxNqxEbV9EKFZYv+e$1sCt~V~Aj5W1{9=oxKq%>IFIpfBF65aWxzJOvz_<^X7Lw z5yd6>kAJePZJX+YB*_LU>Gf-<+HBWHTc66;9x)~-SZZzW=f=i_+;-l*`aYf&arJaE z=ub|If*hvaQr^|o<$Ao5j?JL^SyT0F<*r(k^exawrVJ~tTh>2*bX^?TPimkiX;H`> z2ZSWF55iK@ot;=+yl$PQZU@&4R5r;W1RLe2%e9O;Eq5c0$>>DR)D1MFTV}^)QQ1k#}!^YDy)3Yv7fDm0)UhdxOIWv5y=puY*bLwI|3lW3#90mD^ zoWg;tnND3nUYdyUDxBO>h`>$uUDTeF-?6SOCxSWsG_5 z+x~nDpC>0(J(&NkgQ3lGy%TRsNhUKhHG65sSm$uH{`6N=cHqyI{T4M?7UbxQt1r=j zp9v}Z2du(sYHId4<)YsnHs|wIDE9XD8B$0-Yiq~HMdfeqy4UN#!zpAdrp;meZPos$ z`_N`;TQf_eFB$34Ns(T%veKWMza>W2sGDhAw+$(WT$DptUZV#i0W2X^cI|}6uQx2% zzB`QYC@AoK^OGWj2j~EGXci)9{MNhD}1+7n1jQtd%r^j{9GK4T|d*1`oN;rUkoSm83 z#@c$r&K)Vv^s5!@iy1G!RRQ1DOzMsB0kog^iYZa_?DDk1{gL8|nVa0}DP?}6*NuWK zhibbu`Dfi1@6E=8*}BFC&#P|Eip=0oa6%L>ZaDbM%Tk5N~!3c!t zIDd{kp4OpjI2Bg?{uU|}Ud=nbK|SKSM5)W^&yR1USrVHzCLx8E`xuW-Fs1j8AESe* z72cLT7FRca+*a3_o02jc(cR??bX>xP3=?p|1YNo4H(DB!78SgHe zk7XPfQK{1V`gdT=f^yBu(lZUviq@qwOlPW2cnAgqyKq5JGaDcV40$Q8Wo*|VzA)vP!gZ}nkq}JY? zWI**6f^m|J>%zGPt8dL!wehs|l2mL{3Cy%9VR#wXttmROFjK-+aDBvUIX#W5UF*Z( z9XmHML4zH3cxV-)Ow3SWUsidz^{p{cmR@eyj4M<`B2rAhbsqd0GN${KN$K(PcP9hf8mNAmQZ5S9B`B1Z!sAkmT?w|Sm<%kv4)S_$) ziY@INoRl;)Na{ihOEUYHhvnRGwPwcvAPW_(Y%jB2*jsLxzOm&&JnbK={of}RD(q}i znnlFjL@_E8GbO8i#%L9eMI;}yl&YQ36xeCP|7}daG+0qDFC5L7aBFRPXKDGcSOuPr zcn?-sSXqmUiwR7*hlh(x806*AuU^1|hc&*n>e!p42!3(<{BUsGz@r5|fHvcbhQuNV z9LCXz@Gs?aTTrNcA9J>L~lht}*+!ot*4`dVP)(`a|>=G`sr|X}5|$2MYNn z_rsv|NacL1F@a+cjm~RoH_zFT{GBR#oTa9?p~X>*R5Z$9!I0CwpDFkQtcpkk5eO3T zc{-loAA4VRN6yq-#hCpX8j>w@gc;l%TWeB|x~*T#zsvn4pTb8QC7BZ}H#4C%w=!;j`J=h)PKGW_D8Q zNk#-m67Hb9h!OF}9!l=q+oPsoXuc;UWlEI2x=bVCXnLH1ta|C{XnDhK@<&~4IyuNGH#H}5_neq!A_yhmJ9TmI;q%~hRfLyrz1p#JUTAp`na(v?x~a!- zEv$aj6{43rl9s-EC`R@sI!5I^>l8cN%+w4x5w1~0g8RqY)8w=y7-VJe{w(Jhm+Td%F<1c+w9?n_WiHUZPJx<0d6>5e4O*=dnIH*^|yI^c*O#*h#UHU z{HQDYDz$H#;qsYIfB&=tTR!pQ;Eyve#V&US+-l=G&9hNf_*)f6h%wqo^ zjwl)h-J=3uzIV%3Xxu1tQ_<5_Db6pp{H^Wx?;kwdFD$a0nNjr!BGMoJFPprL7x}+G zwg0QD2q&XNx(A7(a}CGG<)0d%`KP-C8Y6BEjS>G|ORO8a{`ReC{AjAGn%YYsUv=PU zuYZ`9%jrr-L}XG>(RQi1OaJ=NXZ}6)o#}PS%E}63>(kN8xctma=d-?lh)ZqTqoIZ& zqG{f@OHzdBwXk6)D-Lbu=Z0GKkrd*qM}^jv$VMc9GLn#rf#Lq^=rw{&&xSoX<3&Ef z56j8D5;Zh4n++&}8Bvbd-QNkVb-@XlX=r@Cd+d}@ICJF#ZjO=}`;5;#NIZ))7j9X5 zCnt}OANMrekl`S$-djrT@ba%P3f&A;prXxf}$*|n@i##DZr}t8V*!j+!rncAZ<;vz%RW3bP zD>r{6t|8vJTOSg|m*_M*Z_H%KX*GX!>v=4isQBhKpr|q1I-|IH5Pv#-V3s$jG&|ZQ|V8=dM(7&cuEZ+6&TdYNDK%uM>~Ed3-Y zC=>V@XD_(_U;5t1LIHe^eC8dyS!@-N|%dKzqsEEsx zll&mJ>z1RTp*c4l<#WAB;1koQci@$9HtuxN{A1Av;Q`C+uf=Ey}J6c zlX0i}?(CqMKZO`(mZ^AgMuYqsU8li#ZFml(Xgbc$j+#uI$`nu{e)|S}4m7lC+m$IrjJ$wi7SPeCR%x?y{{`GjZ58GwO}bK3NHQoBMaR zZg~35-mxi>AThe~ysQsswpOx#uH|cOwy6#|-7OsFOr)FHlxiojc87GEg1j$a;$5zB zX=&?ylq`io;Alk3*7~NV-qXo|lJbXCX#`)Qbf4iJL0EQ9P8?;;*yQrzIZ^sjNTbWy z5H}@KqlpZ5v|67;WU71qWB7+YnX|i~%w<`1DGNLR3p<`H_xEG@aql%pk@oXbFgOIc zL7i)ek>D^JV{`wI`-#ZYQY79-T%zxI@fd@de%|>%M{9fkr(s6NtH)#G_qHJ7@ zw(le1pB-AL^cIPPqLjsF4VP;{MI{4+{?KV#l)z7bq@;9tMNQH!tHGByQlwGuFk@_& z>j%G#J>RTC6tv;rLi8J4g&&%`Ky3lex9UPx4($@W)RiCg%eIb_tD_Qh*n)inO0ufo zoE~eN!b|l8rR60IlDC!}P!J-hsQ_Ort*gJ?vhea+>X9&hJa6|CgnY@e z56H2!v`FQ1Kk3@5jEqE|iHl>&C%YP1?v#j>`8{F4;Vb!pPLGX|jY~12-Q3DXz#XP4 zZi)@aVx;qMll8R;rAvNLRaHB_fBa&t%MVz&nnh&#JvDc2cAxKrl1FpS772n<4@lB$ z;}6&kiqg{bEG-uyg2%MN>B$=3* zpZ;_fBWPQ)4UF|H?ElHk8m6}{)L_U`rVecd3~hSlUZbT6a>Cye6%`h;nn>@K-e1{z zULR6kb^iYOCubYG8&{PZrjhE`@^sRvxK+i*uHJgvGsm3fN?20Ww0?~cT$Yb7eMb*6 z^z}dow2RBnbTbhNi@8}0fNyr-d@^ku_rfBGmLL4r1A-wO92{`It)-*EBu$Mbi)OaP zS~eC2j@SZNn0I;>VlSAEWxA=j8Oa9@sKpKVJt z@AgI*NsM=Zwhp(bFr1O(R&(z7>+5+78w({mb-K4QWC1$aL`11`eoVL*-pN5K{?O|8 z0qv4fa`Cxj*}oBA-h5uvgvQ2FZ#&vK=9T2X4|!_I6`Yz5^csvz=QA=eu(7q2QU52` z%$Oqkw|pw0p^-`PuTL!w*DJ^O7;wJe;GCI+feIn})bu<*e|haXB%KEVpk38uv6cUB zc!>1O{e{w!vaT^ZBa7S#Wd z4|o&S#eyf9JwcgPmw6S<%F6s?KIsA032B+pF6CI9Jn>wI`#Z$u^pf8U4k|HSzp-vt zS6)j|t560n%xf75)A)aQrZ*3n5P?ol17E%xq5@h;_@K~HSW$7-ET`@4(>UATe+u>$ z=x-yKmG#rj#%Q372JH4J9?w99)NGW{`*1Lx9{D8s=DgI$%JsRks%q$PQGkPu4FEzk zzn#fW17a=&9>5r=;Lo)5`(7rp&>uVVJIbHak}exTw{j z4pr|q=p3b3WqGg8M;vTTRJAG&;ka`>zdv<(x4K+qGK$g4#$F5fHZv5Ow7zIRo*)w2l_+76JFJ8yt_)hVQg9HT(c_^#f$Z%N6xoa~&mIaI9;u){<7)DW$!?WC+;`R^}o z6)=+B*Qe%9Y)iPE&xVFhEF#gJfs9WSJ6ky3*PV3&Y(gJCUb$NWOy_rNz3f zZ|`wDc;(43x3;%1rfbjey}X)@Y?j>)Qm;8}=a2SbY#!=2tS+;Y*q4h>g5;Q|+O`Zx zzL&PG({odvM-SxWw@7J`Jv|gEM(+sG!4CH)1AvE1k4-^Ta9GE>?f{qHXpK*FY&5ac z+OaqdqoNx0lTvmHzx~a^Rvtp@BcGFUUxwVF22NP_%*;fz?cwh33!oe8h47QenE9>< zp8yF!RZsvh@mqbBFw6CHHj*7WOk~ZWmg*&=%ryvb0O9-V5b8=Zd5H2IuSn#7RHbBo zK|>=ZP*O_eG>eM~(3O!nh)8JFaOz5?q|gtR!TBR- zKemrJES{~LBcJHFccx}FqOK>L*_=urAx}`piyn;f`a(foCSKYkRd+di3;K_rU+wXd z+R!^37u)uwzL)3QqN%N1<(1~6zE~F`sjyj@6;u6kKu(6b8QEsAr z_;bqTJKmGEBR*14RABS^t+szlG^QAq3!1)Vj!=uQ1EQ$81yIRV%34U$6*)EHX4`BF_N=% z@GzxZJg)$T5s*_mr+eMXxrG;fgMgj$+VMabSz zVM*IEYNVZY71DBZR~Iy311u!QMkKU_U&|l|D_RzLwu-7cTH33M+N-`>jP#|N4dyK& zA*TVb6dQ`dc$$*m{oRJp7y(h=Vw4rb(L#SfCdIZ;Ix6R2(=LGWtk)Szoj{`ppccn;=E4pE!OPm3p1^YOfYVmD*;#}lhMJLypSAK=K|yILL^}IU{=(a~lc!vD z;#e@o&nY|GY$%O=()QPN0E`eZ^@w+z01&dsi}=vjij6En((%;T<*%foVze=yW%TR0 z<#u{yC6ml6<1VOY>}Q*|OJT78vcvOV0WM@Ng2sb)kvwc0)R0eIJ%JBzf;?KfFq!|P z@lDZ>=H%u|ki8k8El8^iOD&G@^J{rv&^A;a*oO`}R#o#fKBX;CMM*=_`6}Hu5Wy8% zu%C3Fh>@o{V3$Py<`os>+xFJ`B5*AO-VaCR)m2fIIT9mDk&ilhO1GCTf`(q~Q!hSm zy)%2Ch?@+w?QKAYcL**bGn-aHiHwo_?Pjk3E=Zi58;PHYg&Js6%*td#HrdHd zcXW1sIx~-Wwv%6KYi44uwV}-ZtxSbQa{L;csMOFOS+-p?jAfabQcs@IOHJ^!GGU+^ zhSfPHe8A&$<)7)-J_F%|^OYp)$jP|4u#0s>2V1BX6{=U?#1bK}zYp5S!JY5pp+04| zG2U)L(j-45$=cq$9TJ)={ZY*8e3`uXJ3Ie$dS_8(^7y&sS%EZ!?W0xbc%D)x^$A{H zbbsd%9@ZZeKp*yib^oIi4y-#T*Ze-Gp@ucze{Tk!?}VS)EjDbNuW@D-*P3-BKs|#9wcbSJl+~MM>V5S;B_k zk^{_btks4nu)}UW1yZlBGwzv6ODLJtH2 zFk!-yU=1@v@)^Nx8_Ny&SAG?4g%>3~om3FaQKBXgTpI9TbJz%>v>9t=B1#2yWL-c} zS!?@fdwpYr$m^qXf(Cp}dHL&Pfy^cbV#xItr^g;!QbU85wnuPqWAf~ceW-)f+ZVSO z!VV&EfH~H{1>v)%^MA7dWz;6`p>ub?8}1NgSXIC(>^%<-ZQuBKW^r-z^W5kgt;Gd0 z%>3jqnN~+jS{dZPZ+p^?Mk)^0*2^+pMHWU5m7=88m6d8~5kGp7Zsfunlgn&6#bK+U z1t&8=^T{9nZYx&WM7!ot=YHgVp4+X{wrAPnwRje)NWgu3&X0aW4YgSiC(-m|0E+5* zC8eci1~wE5iA*+h#1KY$f+J%UtE&F)e)}7=9*_@uKA*RMOw*t!H#llMZ$F6#K?DIU zB6v2$uO$tFW;E*Ee8cVVjwn)YeqvV7;eh$a1?*t=|6{zPx$_U0qh6sC`B)0?|HA zfK5aK_j1>q>ouoGuWKE~WwYWpp73z-=^1v^2ag2bUZ1mZq$~+M?<%L@^;N+Eg(XD~ zG`bYwEQ5{7&Q{D5*OY3g$z;~aMTiqPB|#}06jx@#Ex^`0{qQ#mNewQfUUPe8(*2ZKf-lHSMK-ADtoUuJe443%z z=(C58U%#R!TvqjDawih#W`2)-;OE!96&Y7iVRb%SLIylqp7DSRhw~LLGa}Hr{Al!( z_bld9z)X>r_u4l${SS8b-ne$092_pJiGN=%)qNbFNexwq_9?*RXl-p@UUs@{K=bly z`q-<`d_RKEk*uq!y1%xDkM-wzkpu}zAgrY>&viST2{VKWGX$!yn_{r)WNl;PR|YDSgFEc<=^P-ne%{gdnVlY=m>!1*C}OgkY=dpbjxX(;VX%Gdux{3lTxTbj z{K9;NL_(TXHutLWv+S^|M{g!-@9IDs7+Bfe&%g zP^_D+lEXl;ZaKMO&=KM4^QjuoJ4`?HHbi-R^kkc2PfFItMRhlq%!mni4A!ne04J^- zs6y;-u&?x5HmTB>RL+BHwZ3O{?&-L6{7A_W{h|Pvc=?4PIQJo@KS_I`Wci0ubflE5 zG||CYxo1CVrlQ^EO@4BDEmE-y3>O00CYvP3UCWF2lcoPJ^1laTc~<2CB*14lo!3KI zQc?PAPhwu!T+)2}=IkmuXrhRrY@gjWHC|)V7FgaiJ!RF;(QNzw>P6IzR-T+`sixs- zyB>?$Arirq$zF8v<_1B#V#aC%yDdEW?EJJBBeUC>e)eP8JnWj&2ekbJ^ZA^mCU#CN}K{73yso2oaz)EZ^1!X8%KfitF)nrvf*Ju1qP3{nc zBj>JsU?D%PAp-S)8Zer{lF!E2x_iJccoL*nE|6Dua;aoaLz{l%3T}{pV5HBitdt*LBaCwzBE>l~ zQj>qXl2yRCz6h7y(wN;UmrlL6xJGc{!Jk5O>-nV6`qXk3k)7^!C#58{x5v68j<<^d5JMI(o`Na{3g&oE z0lVG4^4A-^E%zf@w~N?J((=Js0sN`rGO~o5Qm?$Y(~4h6eG2FGnqyrv7z_6$Ur5h zhwa1AF_OO;d<9WhsLUXzSCchY%mUamT{;pI5^24AVpLzb;^wmlvn0xrV#@?S)(vOM zqT#xnuRP*R{9bD#M*i_Q`wUULQ(il*YrQ&_8LWfwJK_uLrRjU$fZ>_LhxLOfo4&&B zPK=f6iYnX-lQhsKVO**V)GIwUr<#C3W?#I2|IJJz9{Dpl)tG{|_Q8I2^2cSW&1siE zqaQt5A16j;#BYs(&HMy>2>QkKa`^!*Rsg74+Xi(gZvGP`iTjp$d8td31`3L8J|}DH z-NEu7`(%_gby!T?HzjM=)l_E3xU9FAqHJD*;j4EUOcyVA_m8|^dP7!xetJC%UrE-& z|7-WozI6Zc?*HuK45R%gndepkbNLvJKk%u>Gn5h!SJH$EV87!YgkCP(K_d&d<8NcfrYY( zfrqs)pSYn!;{D^x6qCR1)DKK3`{iijT4Lt-ST57bNk9|62)#!>gXqF9=Y(tVIblEcHyGpP@Vcn2|TXApD^5T1X zgmrXH$#xI1=Du{RnTrz`~f*a&RV27+St2G z74$7;Pf=7u_lr*6;Pu%o%he}GM@2_nG9jVYf}@+0>+^vxLj0xg{sJPMT_Opy8W|=D zpL?Vb*>?rx1j0=(_U#pw_pTQgCh$*dZJBX#cLUu~a>a+h8z%ddfC#ZEs3x2~6ZM#x zz449ThEf3WQHaC1C?7G<N@eUI4d& zTbWNQL3cd#)~ONn)^yI0$}DRw)1%26!oRCFXiwcU8XA~b6DQt!QDQSQ9~OYx(nM8i zgj-P3lORCdW*$o%BKy_iu+H(&nX|CBW61SrNDAXyXc4{a^z^P%1U+J`o?2Ui8N5YcFDMH)sk|J*??3>g>}-IBm$)W;p<%i((G~bw&=BFl`+BRlNG&L2K(-Q956!j_pA%E9M zJW%-c(vcnROKeLC7zri0BHG(Hg#*Gm;Cv?s4^1nI>4vlxc;GT}O2!vDu>Q5@`vyCW zFH?A2oH~?Bo;7_tX8;tbW0Mwo>be|> ztcTGGacW&0Nd(;93e^}~Vq$3Ide^$V-`g|4zC10%#R8JF7F7F;<>Vqp7;^X^$F8II z^DjR|y#kg}h?;up(;5jUSNc^`!so_^dAv@=Ng!r6U)fx?WBu39khlYvWv`;$SX*xQ z6hcHvu|l|-btDMF8t%mNWf8du;UF3sS?4c1{3L#76?2iRG6L*fv%~!* zg<mH0H<^r(f3E$SmLC=f++tLl!;}GvA75k0E2>Z z5>DlFGCRX#vy-MQ^R-7gY;3h~0D5p(3UN}RuIlVu(Zz(Ky2jL2LRtTlBhBFw6gOY> zS6zIjpS%R+I|AxfdXX2^HTk>B8-i7l^4~rF-(g+xI((YW^B!CC{tKicS%y4@%-rOQj?0fK!uJ!M+vcT@f>7`>;Iea~zfLNex+eX>1 zzloSlawTxcD*iKMs%n9J*3s#K{pNl`qGXs236C3^?#mc-b^0>G<8S|= zh8I{>)F15`i0wQixcD)(t9t9T;@3wual<^qc}!+hDJb^opBdIOI~6sxz)>8QDRY)Y zt3O)gusELF_~f`mUVDEe>>!R>r!U&?lv$QjU`544V6PPW^j1tvtmUJaT43oLC_VvR zLO4eqUz3S8p*$lnmRIoUYSoK2roW8+OgZEQfE+exd5{eJ={XeTG~+e;h=5)T=Eh;| zAC(?E>Jn8DbW>qv=?WnzMHJ6fykHew#RPa<7x-=KDKL>k0~fuIgaAo!oF^ zC&xGodT8Z6FMU=S)oJ=CPST^k3HD$J?C$P^Xtkn}4UAwA`g*D}pr;+Y20;AsibXcG zc3c1I&~@klH|Wj-GC4E$u=Pi(T$6B<*C5t!CFa^q$3o0$BQyBA)Xmc`GL8rOXm1lgMMC<6;IG<6*G;0=`TM!Kne zWb6A|=K!FcRBf`MZN{Jjk&#pw>1#*Eb9Fpx$b)Q8(X?+}S>PA+3*Naw_F#qKmBh)F8W9d9>;NsHJv1(eI zI6@usge(2@Xs+M`ofut|=?-@A4P?iz-jmoz`A^sO_SN=`Gavfwu(1NlC5QnjFheBC zl+AD))?bYFY^%#NYUoQ0nl)+P1@Hk(Q?txt)7BQpKPAZ+B5T<2tE-yu0hY@1ouL%v z*|DXirR}wSNs5>=WRjABiSe1LN)AECb^o&!XZtFHN^UMbhBh-#1B+}=F|3^YYPsj} z#{(p{?xz(aunB7|@=(g#nX$~%7aykT7Ij?&kJ2)Y&FW#{J>6$c>e!@fk}B!Q1l%EsTq9%c?=$ng%*(>r}&#s-^<<4_HG9n7Hw=BN>y+x-g^T9K)CU2m70RT8%lmS<6!s zB~4-f)-W-H3ipJfG@qQ3S!Pr@`O!2^qqF;ixM|KBkXP*OoAaCbu_VGaZ+B0|d^zJq zL&esYIx6bE2cLd#!0+rI+3%{i480}U#I`7~97<#tqr)=Y#=g3+1ALb|>*GCrOf}1V z8SBzj`iRyO!L%Jz>a7YMDo@z5!=w5qb?wo|)=(#stYn~RIjZ{`b| zmx9~VuQg|L;;w>n)VG!w$ETaASi(m#)Th~WAac94)fG7mp3-l?od2aahgcw?K!3U< zDF<5(u^i_9w{>o0!1Hlm>}84{9KT z@sfg-`EVOHD~->elD84j6o#Y<)+$E|AIp)(W*i@&E1}-0}ZkPoGi$KV?!|9NWG13gbrb_S8|=^%u+D za;lCNY4K3bjanuI-rR?OZZvF1v7YsrE}=iLE*eTtIc^P?Y&%}4SOLk?Z-T!!M*obb zMQm@wWp!wXI#fYAKKI4FoLx=r({0ez`XymH0!=F(=9+ zeWKTcr?HXi$^HiQ>Mf+#8dTjyNpDXfMitMn#u>qdvNPAI>A0E^pO&xmXw}y<44vNC zuL98`ia9r;c^VTGv`0{Gl~FHJW^Z3P)BQbrRS*TTm6i=nkBc*-I@y=?4=_gcXVeun zKzd0W8QFWz1riB`KHrKcWlbjkS)=VHSC&%uwwLHt2}XG~J(a@y%pEv4in-UBp-d4Y zhkiv}SaL`J>pxUCrGODLL==4eYNe{ZdNEpk+1`|lAW5~1?-*{4ug8IWgKFDSP1(NhU|Mis9lPZ+ONOBz%WBX3{Y2A zciQVo4UAg*lzH+nt*se^F+(p?pFTO>oy6-f5s*S^iK61f?yhSjBc~fs8BiRApW3;{ z>-LYJ_(atI+u|7ib+78xdawwaj# zax9pMo5~5iAs`?)ar3Y13V&E<0mdfBlepP6^%V-MiwkRy4Ky%do~-Y(Zr`fdaS=Mt zdaq_qj$Cns&L%?x^Gj-G$}2TMk6PdCF!ZIzvrEF{AW_wXSr5*ud)Eg6VnNzm^SuW= z8X99iA+DL>e#;bsRC#Ob!=|_Ka|<)CfdD6dUaWlX<1U3i}D*9aL|kM(&ZfYu>K(T5SdwLv{`gPa?nj>iLKvDEsJxJQ+# z7@L#`${m+{xu{^ldVkInlzB*`3yL@L&K#P!`^L7q+N)iL4tFM%Oe9{k4D9>|IvBaN zIVIFq6@sbKye{P8G|->@o5C{NC^sPdu9)Lz!*|+RpGS9Af7!hiu>D~1{F)Ek*wg;> zZb(j?v!)7ONX*Il(|}i|;^qN6g=h4$!ad~K{bXv97Kg8owjo{ZL)4G=GN6QJZNqoB z8F)Y%<*M~#H3csQl2#vX{VOkzSkMI&+eXv*r&m2-A$f&_^0w9@m5OBhv#4@Th@}d4 zeEXqMQSD23F(APFb{_VzLj=q#Xp%CsjAgOzz9fety7QlYFp zv=lM)Sja@y^pr1eQ(7?>XXe>QI=3QTvidx2aOr;_yf6%FAk4bSR-~I8o^Q8IIjA3G zK1`ME-iwY2@T_8If8zQUCrZko&s3RTOy8PE$2PV&C)wIk4)yJwU(nPEZzwL=0 zRU`;8{Dx`iO7bkNEROfx*F$y>x8MP2f&1x#HSpsVN-8(~jp&q`{xLHbp=awc6&$2p zzLo#KS%9pGyivCbun6lhm+oO^r#u*%A<+{4|1PL56}-#`6_ zPqXAJl3~gOdCr>4SV|blXxoyM7H5H#;T-{lFjkWeJ5&_gGz;W|BOnRG41voi`(2Sd zJ-OIRo1do+?$q{GB#VXv0duA#S*rmWz(rWcySOSdXa&L^L>mL1UJ3&86w? zUrKpQ7t{x)m$B+oWghO8XJJW6i-%Tp?+ZcDKRu-kQIG}M)pRwQ?y{8#|FTjRDjVCB z;YM_#mAr$tgqkjYoZFWjnQRu=ix4r=Ov&$PFKF~^1DvNapaM=Vw3@YsPc;}bo(*or zekCLQ=d&g(b`H*cj107e1z@vs!Bn<)^{>vJSHwwmYN59C$Gm{M&x7Up^Rh2tLgu7^aBz)BGC^&N15|F+^K`TLjv$|PJoc=SGFWht>~ z6l!$^2}`ZK^vD{D_(4#@Rh^om?JvsC7og?4=$M|f!6uM-eS&vzWQ75427Q%VMXL5* z4KrkF^4#vtkI6|CU>{XWUw?H2&x9rAESfl4+64H}xkjdJ;jL63kZNKf7Lk=YJ$&<< z@|dp(KPWaOEmix3iR;fPNitV*blTSbmxF_?{Dh;u^}Out({d;^08%ut%joxi2k}pY z_@1>+BPw9W1@5+d%n=8CJq;%9V-|Dp8R=AVvEgcfl z9ZE?f-Q6iA4T99r-QC?VG<)>-JbUf_WB1)&*LwM4L}u=pd(JuE_{8^F&yk*B-!BLg z3vboW@~f$7I?i7Vx|kW+kI@5b-@K+LW36c{d?8t0{q_{^@OU;uQHs5*v(0`v`3*(G z=@a+wV%fQ!5Ys0Re|#dy6ybp)QHPu@k|YS8Z-{fsG9^mg#~R}Tx;C9ET@mKVzxR~3 zw=?MCeoukDkr|&H9vM@5IA0RCc>MkS>r&k&HA!LVa$jOAYXilb@2qkDiZ`ES5)gz- zbe(5~i;?8are zAL5{f$qW2IY`tqGp(Ru}=*t|u;p=fT+KS_F{pWavE8CWKfppVQJ62WN{;26;Aaw7` zBH`D~FUTnEOfM=Lej@Y*(5k8{s%K6773 zAZ6tU*W5E2_yAI3a|o6Akay-#wb7rP$b$y4~@ zg*ko?gY6P%+96AH*7E3)%qJ5Sn(w-CHYMvYI-XWFH+PIfWo+UiE6gzr-~q_U9a`9# zKKq<~P{C?Q5dkujxWlfpVUIsP-_bfGSCd}*973*-=p#cBCC*^EPp>jqCFjL}!p0q= zZ*C!$rvT1c&Zaq)F4Px#87JoTPyL&p)-B#;C-^oJj5IAS?i;|>yx>>W&g`UxUN zSOa0;JyAaAMj=>w3Q2f=65K2Fs1Y=HO_hS0OMj+@VM9K`AB_sZyK>G=Ty&*YkVgp$ z+K6}$s)5!dcc^}T*=f5B?o|0z$2w&vg27yh9rFD>kjoT+%=*n%6|+^?^)>gC4kKYU zrm@A|j#ia&Bnr#VHdVtDV?)Ce&yVnQ)ih{g#iT+-&CK#&29jG@r%8c+Xd^B-B_)p& z!~7q;Gg3~GqQpRYe;&&48=97)Yhhy4?0KH~wXI@H6S4dl`o$ANi?A43@_ukZyHljR z8V=&cE|$}vDI_JOgfE5pqinw#z5yO%*L5G8Kps=%CB@#aUzrUAv0)VK3#eIcFvbRC z{bDBA)x+fv9=nZXR+E7~ez%Rj$i(DW@$9;Xt-7LMZCGh8qPQ6_BxrAIj}BSjqo)eR zv)LFy(MOCCI+&QogiJ7{3DeiDcyTZ?kI#*nwx(f7_R?mp+`C-DrjxqA`|L}Ho(YqP zeirT1lYO1Me=Up06DdTRXSB}y%#AVogvf|{$Gj*wP#*tMmdD~}CG)W`;ZuZ#&C}>> z)Xg{fWxvy#+dr*13A>>C@_ASq>pz2FvJ@m4D`(v+8tH>o*aHzaU!vp z)GEV%=g*B|VPPQw+B7maoVmTQ2)K7vo;D_|tHWf;J6q=*2fi`UvGBh(Yuoi?*sH4@ zmIrli7HT=#ZHDHySEtDp1O<=xmja$s?(T%TW%+Z{C!UL1>DYMc0h%HEGt=A7+}zEV zB>tQz#YL4TO)lYiDkNDeeIY0&vILq+?}MFT(}fKbkT>dk8-AXOipc2KTREG_&{%BB z@?#pJY>EKxY5T2L%4$lk$JZ&>*9d5H)6OzTGX94GT{PX$a2tX)Q;U#W!rrF zZC8aPFAbSM(sMFqtbG-`!iai$Xszd7=q<}p3q7+T-QIcF1$GD>X4^#+sl&!a$yf5x*vkHj9&nBZz?=ol5} z(d=lv`tyI)UK-{STz(XmQ$IZJ-J7DbYri+g4op6j2mX06U(d+;YyWuPvx=S_>-irZ z|9YJ*j!Ay&9QtBfakRfg-qg`i4LgV!pgcm_uZt$ zw?Ciz?4gJ++~>Vij;!H&Nn?q zVERMxT}+Y>hyGj(?dpCtw}i(;CaG^aLWoipSHSgKW!BeBeQ|MdxzAhsJc1VN1Q2%_ zk{d!SZB5157F}!bVcXg=;RC`bDD%EmY5!edGg~d_AUFUKBxGxM=aF~j*IVMll2(|o zZ1VB&AzqVXjFY^4O}N0 zYb={robCYkub;|CCiSgnalou!uWFQ{LWBL!OS@}K_9TxR8MN|0>%_#Jajq6iEF%JT zc$~`NG%=KRbKwRLtk7{63%<~D1+kdOu}j0H)T@N<*W}c;6Xm> z2Q<_9@l%P3Ga(^T`X#$R^w7~GHGdw`(v?9V-)^V|QyO6F>o`XYrMbBqyCsg-fJw-7 znOs$lclqjVa$?d`NLA!WU$I51G&u!Dmuu~O6KrXtga?8~gppU8%Wk$(`>O%=4Q_ZV zK0a>{*bWJJ1XWZLUEcM7{kw6|!CO?)(n|$gh)u z@f#UoNo{{l2`{#Pg~YMx3GVUxyy!j!#Wp zZb*|u42?E%w|AjEJ?yg3k1S~|AyZq`d`AaO!v|z=uc$a=)z#I-<;(kXoLHDM? zLlMM+{V13dEnbQ%AHk|GA;8-Se1@Cz z{ivigR0wnVakBKz?ymJ55+Tn_Q)VU$7u$G6WnEKB|Fj)WVBkOT;P26(936xD(1Zgq z|Fnx9r8r@|aDA-!wme8hUt-E#}HhpPBl~xAhM(k#=;_U zJE=m2L=QC*)EhK;E)3 z1JNyy!-y|9^7y>&BYB=b@uq8@1kQ82N3B6Y7{QVf?u=S5o+I5aGx+>nZ?@#x9`FY+ zLuN!Iyk<8RLxS>Y3re3>{qpxp_o$W5%UZk5O;j=|6gxqGBidc=J)PY@VQ0wt$^VQt z&Wgl;cUT}K?EerS{~yI2IT7tIX(9>=wkPf7KiT22*4F80=~h>x26QhNsHrg>nr5sP z_V8rFUKmhHEvq*hjm~&_xY{ul6c!S5m~kI#2n&1M;yM)(&tqa@?QWlqo%LzEJ(Ly~ zj}6JCz4f$UDk4@;h=iJLn+)oRNr?0Cf6C2kR)4ZI%9{^2KridyEj*(Ig{=K^AFg)|IjTeA(az%BKwmjo5@3hHQG$0C-4es7)G| zC5zKF7jmrO<{DC2ORj(CW%&VLgOjqHTz)}8fA_$PM!Q20pm?Dhg9%(c4wpS_1O}@Z zGg@APbE0AKJZ=cb^#uhqtaLXOht*Dp0!Y*E#pQ=#D+wU#2KWk?3kQOL?3F$5pHURc zxHDTe7@AbWT2a9*=JbaBZA4tu4N;q%G)kylUidwile~^+Yj{r-*ceU^f&-ceK7x&J zb`2{&1>mGCpxs~E=y-mq#Z}gn3>Kn<|JAHLtEiwjsC)9nJ8<>G;as!JM5726mh#** zI~$v^p&__yKTA}KROv~-e;m{en5k72zzBRl^TXAz+M&gSNyjfhpfqa~3lo#RPuj?k zdtzc-SxKo_rzJs5JR>5GN&S7^WL%7DRs4P(vg!&3It``-%t{-3AVuWS$h;eDHhK0G z=cUu({8Nawc1pr%9yJZE6>I?+(r`h9hJ!;k5<;{;F$`Ki?fuRX99$fSEJ>^o`TD*WL;cIm{@Vo21n=m>?be! zV`7wH!fmHWNaXDu)m3jpQBa)Qh89%#MC)EW3|!ZkW=(>k&lM3dkd!e@lcP+x*;rUe zXk?J_Z1NHh=dJIMm*~xN;hgMjgy&l%&aj0JI!}=OG8q|lkffRj8yMJ25e^EmBvhHN zkBn5DF2mZUWdsVKz3JA97(Sz^fMXCH9idk3sAynd5Ik*cyop9;F3(&} zN%_Hm@54fqQ%ipx)~M-wtMX$+qU@Z9WM&a*<(&Lla%P$Yu{Y^nx?um+9L^+-kJNrl z8s4@az|}J|+ityUczU*ba|Bh?7q2PRo)fL}7jR$MGkoCtdl!c?gaaZYM>bNl01#7U zUT5MdI2^=AQ0rq%jC5j47sud2h{c9mcGqVEC$+Bc_yZeiIx1T-;#X99{ zYtyH~TFIc-4nPGU*StSER=jjg+6!7dA4p-43JIB_H{SY9waKJqT%q=N8Mt+hHG33; z^njJ6lJQ0-{PyU3<#?F37Z>-G+WA)V!_qMLhC#4-0^!dKd-N;84hP<9QCTS>?QXMb zp{W>HFzVa(1dj4^ak>2ILK{n^tr%KMzW%=8bxY33crP9LQO`IxC!_gcXn2E}8uD3R z4)XC$0_2x*rSpiGgapFE5e2RIUT;Aq4q%DXGxV*mudkq^!)_h&D=H$TD5VqLwst_j z<5qBcL2EJ__4^Br%nJ)Kk){_WatjKe;|4-tH*|)d1G3@%REDsg{BTF_uy5H}|i z?fat=R5&)p9T9A~8yT;k?M(YfWT5rmq}09ql3SmMZ-ObOt7>u{M1jMo?QusL86_o_ zcFg6mXQyOuDQ)@S*#tZak&%O65OT-HCy!in15edQ-u3?Y$0{lxjQfLfaF}k!hqrn4 zwmzD4A+GYSt*=EYjzlR&YWCH2{3fL%Q&iNZFtxP9Ha1Sg-&t^jAqNVIlaB-$r9uk| zzBnBm`(}LIX#pV=9{w&*R+p1|%KVc#Fm%wVT_M1}uRX^yc!TTmY~g zTmQRV;Qg?*V>&)I*6ezBF?S_dZGYX96P%oHi$^ev@5BCV$}2y=`3A@NDo=1LV;=fv$hyu+G9kLW8?}JoBm&-skIY5`J-UX-OIJerRtJv%|=SWkXPKaBmM?w_)UDuEY2^ z8$pxfTXuGtBwm}{-1Eu}ph29nGakgIVdAfa(vQ$EZZ_rE*_a`} zhJXbh$cZ%Ob~AiZfXghuSC|#^db;2Q*m(Tzd(ov~7p*OhF14YYUc*lRxlRF!T+7)y zlg*#~+uL~jIEF$w95zUQfnK=iO3DhL3h_6iU@WOP^h9(^AA{ z+3RRbta;H;Q|3)<)PU^h>>R(VlHE8E1kGJLuaH%JIacEa(rAlM8^Uebm|2n=OaqHU zEOvJnp;KG?v)%hVFi_%^sr__@wz+`gkn#h!rqBD=1=KXTCWbysseP5uvjVT@KMtI> zx}hs8`bmQD9F}UVZp|iwf~1QWg+EKJk|wV&FWa#`vC9?c`y*Vpu`bg{%AsygXYF3I z(GmOlr*XUKC@Z^MgqZb61nJB=UFn=|)}G16lE{vL z8lo4(lmwuedw~WG62xRcS3*J>$ohWj?e~D5f@aO3R*OwAZX@P_;MVVv3KmH1OEgaX zttELff19?UI3`^TABFEE{QPeBJqFQ5V3BJ#s5!E8%vKn{l4`y~(6ER;S($U|-Qw!w zlR&g;*Lv^r1;r6jASWj$-wKrY>YbQDH-OW~*hPc_a;e?o^7+9^87ro6XGhN`1>Wai zu^Hp>pORw7yb$E1?z_bW8N~Lx+?)H^W|fk?J=|iFaPOLyhF`UL-+Sy^AP{rgX8z2H zhM-|zzoBF&hA1eEA|WAF1D}|8^5PjFAETxGnVZW_MiwQOkPy>vYF=1SvdlZxdZ&gE z)mlaL_MNaW9|c>e8mOlchF>x1exOh0kyI4k|4sFoh+RrZ$hP-FTh~~j6An`Qf?x%*hR9wa?GTTMkLH6k8ZbaS@C?uA9K4!%`YDQWM==~1oVVXjnz+XD=o zNp%Jj*-|8O29nqez6FoQ6ixDbnsZ4}MTujwBnV}kZ7n#YJG3;wh(D`TlX3v>UD7CV z2(Yg<5N*iG&DmUE$0(Z95(R=!8X6i<>LGceIl>k^*pnC-7$_PC-*>fx3N6z4Zd)-r zW&aoUE(FqUsL#b!r_P!?HaUrLoVoj>H8zD|gHfwTzuaZSt+pXI*MyMI#ou{$Xy})j zj-UoFq35J^r1+YDC&H5_exM|EI-Fx!wBxcFeJAG+d6(w}Jq51m2-o%PpK8jZK&^*@ zat7KcRSsRLK|!>sZJ^zK2^({BXys<)-U8QIuOPeo3y8|K^7-4_`X)898hAp@s>?;j!M z9E33@d>_xr&p>uMG#u5g$sXKfJFO4gD*F29Q;`$XE0BVN#K{%h^;tI2;UMZM-)$HCnhpsX5R;J~6SQr$K^=mFe{%t1WMMZf?qYyN)7;D$`eD^2QB+nIIGyv5 zQ+rot*VaCUxR+g9&o>^lO|OzJhYJajyj-MsY-Va25gGehM^vPvPqXSd{JrfhhjK`W|S!i%viOlBppa5Td)1Sr~)nWK}9`!Dl|My20@3E7^YCHn`$BT7>jf_l8iw8|* zRaH{5qHY&yqc9R~4P|8t$`2HZ-H=z5AB>pQrLcRahL(~J6ou(th-$gN-gL0AXnYt) z9(Zw)%=FxyrPvw*5vQTa&wuQ_zu?x<(XqFW5K6KN2SGfROGYk!0d!9-*%}(W|)6Thlb=~I0v$D@x?rpP9 zKpb!BNq~oke}AdH_EdiY1!M=Md;uB0VBH-y-59MKKGlFd;=NMLD4W5859h7y0lWPja#s*)Y(_l50NGR1*ytn)9jm8%Z?k09R0!aMB z{m1(gw|D9hk&)P+?`#(rR2UG#s0NdR149ierYJ-@c(ZyfyZ48qOllq-R!qyu%eP_# z#|=qHn&vom zL9}2^scxhD_@IFI?f$rF|MF6PNl{UL@yZpHmKaZ69Wn%+`i6Vd%R~)Q*Ici z0E3o)~)OdR@y*JYTQI?{>8=_)d^7fxrf#c(Jki z>ZmRqW2dv^mulH!J?E@!!duTaweTFeNhD~tWpi@;ZCLmZBx~5usy8Hidq4F0gYx6m z$}ATc{M!H&a@cZmdcK0ba*l!m`K4bDd8eR|*&$6%P7bp4ii*pw&nY~bm{?eo;2af9 z-JTqTup}5xK;O%)J%zO6La3Z6GNVdkQ(Pj~H#S)l=#lM4v3F_2w6xsK)URrXtwwaa z5^8C>d-oUW*ywoNx(+--RQihA z(67IR3i2x|Zx9&ifnq>I5k;L#9kSH7C8waUmP7>so>ahf$k^BzF=2V=daj3#c65Yw zl$FKrwjRSzDt3G5ONV}&(-|HY7gsacC`4{?Dn2$fAvP7b-S;8l18Y;hxU!AgtHnkX zNNTTY0G~U^&ZfDrQ~EDAC>aVmzN(K8DU{`g>S#=$g7QB zQ#!bIJDf&^1casXx~^0mT_aO%aXxzIYHY0Gy0M%^g(tOPJNTqsh?FlO>qq`KsG-B* zOpHn05e98b`chj?ksS`bvWm*iWKDRwlm-(YU2ubRA9up9$*v7&M$@+5pitQoyor0VkL?HrDyy_0|iwZ*xM?h=SKz~(I!W5IGW2Qdb&n$11Y7hb`sS0WR7(&Qg zRaLAcqhcjiNn);EK6g4W@8(c_2J(9eN%8r5kGM+nlk8)C^c4_Oo7^t8#w%}@4e}tb zxY-#QNr&xP8^cD3!0psZCIYFRmu&Ue@##ai{XqTyDRn@k%n-^j_=mLa?My>awAqrE%7y9PnW?*!>&MVI81 zg_(trWr~Z7qPn6`LS4MoW2`%IhI=J9JE&r0{WK4DAaEAHOPuprfSrTZufn_|5;$e! zhf)GyV$ZNFGt+%Bf-Yb;tsnvP*GuD#E_5|Y#m-;r(h@n;EUwufFTPP4^JL&QsuxPU5s_P)INnR zY*d5#q!NcoIY#+DnR&gxK}_E+6qYFMEk#= zUg*^BiS`+Pcy{-+dhktjzd!Ih1PhK$*$A5ebm%x+hLm)5y^gvA6Gn3_rfdH!@531s zX$Q6gOI=N#3y&R0Z0dwI`tnne`#8yD3DTPf(%YCu7 z&BRJFpnIZSZa55bq9=ZShhzeKGdHl6Cw5>U0#;hQJFx0I$b3SR5mP!_wGDP1K>edB0g}OoL~31+Zh}&c=!v0v{hSszq5p= zq|E2@GHRNbq;6$V`1BqxH@F zsbJMobUQD>&ZOMcdyIW@%d4pG4wOWfqZ>FGnZMDqfVCuzjplm9?N#(S= zY<_3%2Y}Dj$defwabw2~y&&Qeyv>FzN@>i|goUSNHo*?k-Z+M2`2bGQ{%Ks=^Aa9B=@-^Qw7__SUhG%!EQc+mY{V4L?2NxuEYtRhaz z6+~}8&hM&uMiA+4T&LExksu`$Qm#d^*3NNhOn@uC0Blh@2S-1=9g$ zn_%n@9X;$fMEGa`2T!_F-SB^XCX9=RcXD^Q-y&>fY49=sf$`*SFia5cou+6*Y1sYp z?>v(D&BGyY0D;dGdn`nYZc)8t*;QSutCk*WdMc>ZVtsuIhL~X3iFl?jX21auxf&1r z)?dWD1wU{sXtC_W-+yd%=^OluyWq)(jz}W^2aHD(Y9S{07qoP?yKMR|x&z|>neK4^ z{)7KDfJ*Of&Np<#a^$SEn%+1~*uPXpy0bqZ6Mg*j8U&)ipZG1xvNzH*YfsI+-2Pli zrvXLSLxx1N(l*d^cWKX5S6iD1@9MjoUfO~R3kwS{saON$%u4s(t$96!uENd8Y``uAQMt2S&XagA^oIVe0PGN zqY|oDi+tBaYOq}lbj{MPtnXvL1iao3UJEDPfX3spQ)eRR8$mwGWRqB&A?DRJar0+oR*5A#dgsA)J5Y)<{n17_Ii)y@Z|kFb@lQNip4Ug z!_n(nX zr9*lo6bb3}P%b!I`qF$s&hqqT@B8NhGrV06%bO0riN7GjRWS0Mbbdbd-QH{8WYcGN z0_KBy`JBpq9?`oWO`bAHhu(f|`-h|OC;aLz>Ohw4cBmoATu#I9WW!Cwf9*H$=Y>;{ zlgm?QTWQxjZc*Re|IW+ml8sSSdgPtb|4SUBtFWj5pgwb`h_F*spfOo!83Z`iIgem# z@`!fwWr)c1?2E^*_+0J@A^Dg+A=`?{dzPTPdlR14-qBH~IP<(J^^H|XX_$(%^nTH# z^7;1`=2+qrU5TcnC{r-Ins*B+z41 z(J?dc6yqtnsh=yUsF+)rUS4utcb6-go6~l^*k5cq{587C1oshe9nx5HEzV|+?^mfH zfA(~C;A-asa|$hE^T&fU6~6lK(6y?NK7hIbciLxzb-6#WA>R;+^F~XG>*;8z$@6k_ zTZli-x5$PcXoDuO!794r{=BZDA{})=!YC!^L=4QoYPZRVfMRrp6R+&$r{}9FD4=5! zx&lf?Qa`1{?f*^=dA;)O^bF}ww`DZ%Y}=B8fgvU)s=;ou`D@#2@KX>7FZK;6XlN3? zCpb*QT-O4^As=~6f9+sNhNW_@ezZ17F9Vb0oT;d53oCk{(}KFHc?IRidsn^Tsl2O( zGC6XD`}@QK9x~6rBqb#cq_B;?)8!CodGq!*BRy<~TYu{#)l(1>X#FB1V?FoU69Cl$ zqSpA?ipn7C+obhV6vFg^j1P1S4Qso3StV>kGa8D*B|q|=0YnA5!;qz+MYlrQqFRBQ zIyZb&6o=|&mFMQ}^oRPE^K7W0A7I*F3+h#rAk_b1Q!S<^`Hry_&|}Y_&(f zKZE=*Ghix+HiSnZP#uT&K*6 z88s=@s~Dk#vo1~(Q8#tGgt^<>;JDJ}^&rkVSbm41_aWQ+Zd%CaAC-6l6T*^;Syff^ zmx5EE2MmVG9k>{?o+R=0)*c@H}?l|1{xYX=ihOhdPz~Fs8Y-PmaW3pi|0QxIWIlT<#E>t*QI;KntV*fjBejrX)vp*1QLnK}I8w zai~NS!iY|gkpLd-0U)STb8y72vfI`cnB~nRa<6<2$K4pYegEms?@`909H496oNbHH zF$s7bBLzz7lp8KL)>X!qs;#>93s5mxcaDiC)$G$UGBVPYZM|aXsPvthYnwY8`i}AR z1<|CO>TILUej%3Rg!N2rNlCTw`iP|J?ZSJ{ttmkbV810MMH&SqN6#|_8-c&6cp!B% z#e8FB@96$_7ib$o;#u7~<#6)sa6A-8M?#*XbcS9mk?;PueQmg3(KHxfiPUd+v9dD? zhR~|Ws{>{EPnVMLa9OQ#WB3?aW7G6OMd}18khOuqO1VFC!Exk}u=&--U~W*mQ9(;< zq_>djrN86hLJZXKVr#R4}7ifNZGTQ3n7X6S2S@x=>v>{_;nY zQ!ZM32&F8(z{KzUw)G+?O@TJI<}H^8Qc{1j2h}&R=`z_vR(>3-Iizg4?i-jhhG=p_ z8#y}k)2Idt+jy|*t*x!yy5TxLL6R8x$OcN$`1n{@+DSov@nU;WvB#pxjLAJ~p>zKVI zz1wYegU)kOx7k%VZdXNR$$alz%u-Whm1w1CzcHV)40bNpJf--*+we8@;hNr&>mdGAg? zeLGV2a6Q-Bs9Wj|92F0Qx?a~8#kv5(WT0I}RQd31&G%WS0{~m;lv^A}`km@DyK1Y7 zl9fQfX_;q+Q4EaN@gJrXwjXQ6Ea!>@m5p!EE~JBEKW5 zz9Tm8e`onsM`&*pVu3HberG2!REGzjfFMsgwbo(uTKyCov5;XAxjLPnGC$P^K|QVI z^VLzgVmEKHMuhs>#Fa{+M3uPoF-4#uiYUT(++P zYM@L{ak3W7CaNx&vNJQ4|7J1bL*7>F(7-wo^WW-u&7kJnZC=NZc z1~W4Ov*@Zqk)~ik=HlAb6>{`?1*nTq0^euKQOBnA9|rh391f!l_*7LzyX47uk#HkH zBqV~s$V0?HaVhMgE0~!emmU7m@4P`sMCx~S4VBpFWU2OA8nf;7^%xEO9U)t4y0TZz zsP?=1+k2)hfPMhxhlqQ6gRM8tm(MtdCHCs$;>q9mzJgqA*q~qWBLvT4i149 z4aE6jISOQ9OHVWW+GoSlo}yH#C@8&}c;p=%B@8P1VL9QaJo41hDQ@?#`S(^a&@BGTXAsL5@ISkKy7OddDTfp>9B1vz#)8)I-zCkC}ZsS z;+UJ6%czHB)xp65up^@X4TzC;>cjnVxEw;q-|xF(a=H)}Y7-45M0n)-gpC6gVoH&d z+}~0FwjkE^(J?M+4WMIy88yK858=fYiTwjyaL2`GCutdRK8K4(5P(LBJ4K}O+2JqV ztxI+zQd3Z@v-dc_7U0{zU1GKDu6EX{YsA~M$sO>SA7h$&qJZ`ZMumCXB{QiWYM|Z# zSVlNh0jcv1FPYUHl*i8rYQ80dbHI|v)H^6rK}EYXpy3lhUO|WYn=)7oNgTJkvpA$n z7b#!huNls@UNKNyU2X?YTEB+#mdKFnXy-Hoa8=7g_68**<%e9kZzaMU3Xs(7)zwbg7=UtxccFgLH>V%9b+5(=X!d312djNo;6}rIgYbn zQRILR*i5A{%ZT~x?!}B9Zcj-@2Ar6s7jf|A+R)e#SPH-Y1M(2C!4bZ+tA@GBNYDyx z1w32-+J?NG0`VM$s6@Yz$A91pEmgghfr`{p_CMxf+1Xz6rp9k@0nHfbG|dKN-6@I?Oz5_0E zPURCwQmy!!+ql>6h6Bz3F)E@LCi-4}2rytTc(W%>!j2^SF%hKGW5T8FRMUn;4{ z$mj?u1ogk#1&D!~<4-WA!Q*U;2C}ui#p}{z(j!QR$ORWT{gPm^WKE}Azx?XpWz{w2 zl9uo{oUbb26#)&m{>8)QBzzn3i3?g#Et8-<#6Z|*f|tI-u>69A)AqJ|5y)DKCM;fK z%cSsrce@Clof1DkKVS4xu-luh{C&1t5FH)agSW9ck*EzyyeE*B-iO2)NkYu#>d;_W zQ_l?Sv93f8p;HO=Kk#8|jTYm(&t)$VJ=Q*2J3%+sqVkxCef?^jyOL|MUTwK=4qqcb z5o+;ykifl8`QzX92JbnrRi)F2bK(5br>TIW10Q0^D zgjyLD$NR?V_7mUsY|upp+%&+)6YvZh=w(VDHq~a}f$a_7t!`55dnzkQ?XBV2B@^!X z*Kz_puiU0LY%9JN+s=xQljy)!DzU~8G=h6)YI=EF=OrN~HuZuK4GTl`xrn6L zHxAipS0R~qnL%KD!`&rK{gp;4pZOiluR8x{x+1T?q&@Tp$K7ePFs-TRN`WRnFi8xN z5(T}Hm0hi&yNF3^GA=|Aznj4^ec=lK%&21=T60-k}_nL$w|- z9WW0|LBd=_gsVKcL%+rKbfTwF6n3C3;1P!4jhdEO@31?Pry9P=M8?TAIW~r{fTFWh z$4kZJbve{p0pEu&>)T8I6SOEq*py>b%XAcJqFJd>O-wdjEf*=sH&xK1eE`u#=Ecu|uw*%T z6(xC<$KEF=K@}%Kf3T`B$Flr5~%{ zuDU0H87odxFkEh9gBX~Kyx|aiAg3e0yt0>GSXf~&*=#-Y(!+xf)QLr&56`~@_zTz* zaJirQSKo73M~N-1to>3Kk>cO|3-TzbsXc~($u2(~C++e`&q&a{F+hp=Hi)8DJY}P0j^58Os~o zAyLYJL*BIi3MOBwKCOn#>8J@DiVk2RZ9l;T zAegNUuWq7*qSSM919mnOnGQ17iqqGxs`Tnr?$9=%>7k~jcC8IvOR`wlO9%}`@tlMQ zXSpnq0BBY?ry;8*Qr+A~U27uFtY2UIhY|5y>`9VrpLd@X#CF> z8fGpByi-sON=Qh6^2qIW3=fiDRz|{^?nPxF4Ah>W#|Q?Qp%2osGvl`BM=F=4+S*p2 zlJKoF%By_HwNBI7y!RLuaNjfJOX0Vw(o|F&1@)<-8{3VO(a-MMS#@6`)mwe*$sc?LdGuRw7XyF$w^(9<-ozly3siWRE>3V9xtd9 zzc{-(o^LQqX3cz zLF8|(Bk~Np#8#z1LBsDE;tIPT1)X>O&b9gU8<3o`=HAQ8 z%U^A6SAx#Xeey*a)iXqpPJsKhOMs8S!zG2wky~F#%gFydK48W@+&Ar!dl_k)>K~fy zd2#Bkt3fWEA572jSY2m@vVpJ z43kw9mLxVl9v;Kd8cK_2y7fF6>0|`Et`3?|J|C%k_e00!Pbp-ze$muyY=DO1A_nA+2lHR1>{@MNa(= z^z5$+Q~Bzzw7EZ78DEp_d>ayYd_KjHG~xgF$sctC)g9mf1IST@QCq4}yUxD6<z7LcpKfe;VdHqzq#Q9&=Hlnq zkC=HTlaHrn9`UHb@oJ7`@Ci^X z1+MevfZl^**NZTs{nGUGcXQSw!fVZ-(jx$^!u}T1?LSru*ez{Htkpl7XQQ z^pUAJkNhJuUjxgi>11xlqlNuloWWFFj+rJDdA4imXWQkFh?FSUUPni=^STj`nsLfr12N59 zr5neA6c{-4bAL`zUL6m}#~Vd@u^H46|D|F8^c+U@a>vco7J3Gr!2~uUNMc-@VY%b} z-OcYVS)d(1#b!vVF;zEK2|G&3{M-J0;dSYJQ`$5>KK@VR`PHkV1EE1IejFyiTU=Oh z911socNUCzSPuETzT@D*N3`pC2l+Sz`2@hZ-TN9(v0<8m0*N3bBA%t^@_!Kf7s_K; zm+E5Vkx@V~xe4vWgBI1Pvz3 z&*OC1+sg#;{@T_?$|#eFZ%)F0R~ajl2Z{*R#6%#VWT~tO^eg=x5IbZFWCE`j{C$1H z@wuKuv;t5*=|~u>I663>L_wD<7GBH5KtBG{!u#Du@@L3bDQH~|9?TQl`3-M9hy?Rt zLx2o~gwGR5cYQgm7lEj1d7$ZOj**c-_8CYWA!6d8Q9gcZ^rtnqU!5s zioS8Q&u%j5^rvzutBZeE>De>@fimtxmPg==C#I$rk0GyWcs+~r^1yg4u+)C3{{eY= z6q;jF!@~R$3V^0Pnc{!6_tjrbzVYAKilTxj4LVdnO6i`+2nk7PM7lwm!2kt`ksn%G zK)NIci~&kax*Im>?%3$><@5a$o^zg`o}HbuvtRCW?)$#3_w|lf7~NyKXgclZ9_(OT zTU*)!TO#McMk6*3ke$!-29UQ%Ts+SrI{@C}zO^dAKQOFc2jhj(@D@FaTj05-ZhTKh zZSpCfn&hej4QS#IK%gBx8&VU39|JI?$M_c3BHX7~^Kk6Ep6@0u>gSq07vT{ z#NWOPdY6DJS;4^f-3!Il99V-G?470>v|P+d1x4;ase=Pgq6@dsRk@%?3{qa#VmVg1Aq*{je-J!lm%ATb(r}63RsQX2z@5zFg|`C9-KH;t(fJN{WI|N+j8@={T;Aak^j{WoR*E^fW4XH8Omg1%mReHakJ_gh zk}7*c!fTl7klI56V{5T1%601zp+M~h7suu$p-&nI>7g)F+hK7Fol(c zl|}Da7t}@gGTY7a8`E5g@NQo!KLQ#C=MVnR<&r)sn$-t7T&XA&P;AjCubz;KcN16)mNr%zyoV8b@ zb7_O($s^TAc@Iz#IDu}T2`b?9)m@Z?1Kt^&WHT(NrizbmgFoFfybo>&VSX30@y>&O zTVjjfWUK1r)TUv%Oi2p9}q5_hxV?-)IAH8*pj3x8>!P+LS z;b>#-212l?NKw-$Yc2NHA!a@B4wJex&GB(vg+&dn&J{aJC{W~L&ZO1V{RiS6$oJ1t z;T3~8rJrO_A-SQe*=(gJIO`M9Lr2DWQ*-NRfN$CS-P$QJT43phTVL_^Y$TUGwV!gE ztXAI{JE^Bizwqo%0!L2i#lmoQvS3x}OQSGXOx^74ETEiu((iGwu|3J$%Z&h|cg>oP z0W5E+2 z6m$u|m$qyfyRZ;RFYfd$pf*we9BJ^liqrZQ@zh4pUPcCcAIL-(=^Rwl=C@yp%*uY2 z%?yyAla+3(mBO`fczU15@bKMrynEeRtTPpt_&qY4k;;`WEBqccPI`+yuF7q7J%A2% z`Y=)tW&H}V{5QAv4W)9xn=##mPr8v3k7KdFvnVuT*#6o4DXoV@X3o zH?e0{_%2Brsf}cjsj&T+_;}J`{svSsuK2C{CFX6i9i01|VEo)+1fGtPX>6(-u23j& zCLk)rx0Rd{qru8HwLB{W)00_OsYZyOuBMX{g7aK4@q_E;+B75Hi6^sK>EZoWj<6xH znfTC&p1Q|hb_fSm`|KLo&sET6#1Z5T*@+Q5hddMd81kx}*+4NzpULB5U;6F6sFNV9 zYufzD>F#;*=9LK{G6(2@+@z2Qvo&_OxQu3pF0eQuR$en*FUzvSW|6J8$z+CY=0V*% zi3fZrzKc;vW=DToP)8Z^TPSUnYp2oTlMbV$1s7F$!p- zXtnlAsS>6G+-1Zg5ITliK!wLX`M&YPbWf_N{dC#V>H!^$lA03n5nu9(IjB^-LJ|@s zcJG!tBj6j^4A%5VnXrs2Z(LGUr@Bv07+vX37O1it@K~4#zjxUm$;ts4m?L1Wq#S$F zADn(|E6meunhU4lKV94qJ@J7m!YJQ?lk-9?;mXS-vs#QD$*0S zcLs+l`5?gI5Frs6XM)?IYEpTKkDuS$-<|kH=Nd2(KoM1}9KrL?of}XE(uQ zt3(3bk)s)-#AW{k+3VbT1RJU{O6YEAG<|6L{&>^)5(hE*zSPB&Zd>{{E!li&o>6$yN(Zf(@Gcc7Y5h{wTx$Q8n#{@nve0TXeyj9iGDy{oVQxdb` z>B0g*DcKNG(yq>2TkO2y7C2GL}*-@_J>&Jk%H(l#x zX#e3{)R0XhV5Z>P;(cm5j_8oOF_%5~`rvSmmyD5&rJ7LlOYdai9;h&PE^B1)d04H9 zO}`?!l!GjeH25wD-Y>;W!R#4K<2gHsUldVcako4!Lst7MWwmw3rh}o#<=MI0>UHQ< z(kigJl-cEk5B}aEH+y8xvRLnRQ{Kb++pHlzF(}tPVj}Bkl)Dyd5XqxMLw;b>f1yi^ zyYWgAHt(}W_)^@k)|cn+OHAc~TcNMpZAo9-w;b5>nM%LbD&0Zdv9#kQP_5?+S3<~K zCO--=yeR3u4=ES0QJL)9{d|$Zqch%2#J+{U9a$vvH}*#^D{im+mteH@Vw)QQ3A|~L zxoxJgM6EkGW;sjjx6?zZX=y3HQxct7apnFSexZLAajuCXBP*ucXLwmtE>l3=$Zt(l zx#1-`ssojK8i%{}K8Xj8_}G_Kz-!t0s2dx>s9G8kS5rg$@cKKp|3QaW5s(4^v;FF= zaRWm`s3;3|YjI;PEteuD+5vy$ygI*A!n@>6WQ7n#+Rb+7STMW0q?MI?EI=yFp(jij zTEFU;Z(;@?Us{b$*D5bUoWvVG>PoG0$A4*s7C(-dEHfJfXx#Yo%JS7VPL3{Mon@pN z<)mQk8Tk6~OCFw%F3;)PkiUC3goSatjfFt>KlWC{-NPH=5!kj?{yf88vu*eHREXpC z0OHpcVO|44he?_Tblen3Tf3zDQC}K9yn9zN+tSt5wb-3R0nXRnj-K(eWVH5EpmUQb z=vCKwf61Q_ThL%{Hk_20bYE51GjR8FW#t`6`Ago=;9y!_^OlcGk}Q~aL4neFw}y6a zf0#D1pf>+iuLG;xN#I-%{R~c6ayyVi$;yff3KDvzPh`b0K%Fh)%a^)3&!fU%m~m81 zjT>hu>4LXlADg)N+{_#cjPi83y))shDC*3!eR6b?ki_#_z|NYEk){fM!sX0c!4}g7 zyygiQsJsV?2LiN#qAGUTpGaWr7!Zsn`}V7U_tq(}jv`4Dw3oZ%@Fg!|I)mk;zQb2m z0+eft%G{Lbc$1;#p5)|GVR^8pCJSx+VZ5XR(E2F9*~^dtuQeP7Sz=|W_Q$Z)ubZob zXZTfAyV>aPlKr1$XsU;OgKa)b2gM7@K=xFP%F^_axu#`7&O0T>X3<4Y{hi7{&u|`{ zl^gL`@8vHf>6sI4D!|dK*q#nR5hWw%K=m0YbDcwL$88MkUs_#3;=F>Xsp+$)&*S63 z77M670unUOy`j9%_%}9~=>+odH5ms5u5^W##rD%8Hd>tMPcyp!-fcrm*PS9U+|^>* znnVU`dD*p_{)pddig>S-7s+i%8t_J3T)b6BtK{gc+^4Eo$>fgE32Lu)*ORBa+w#EA6vhDJaxVeVV^9NwJ;|6c&v_1V-U3!3W?pu2=G0&y2uF#j zeUf2BfixZXS~lsn&PIVOq8mb$mqUyx`BNlLRx6-DOaSHGTj#o3PpFG~&ptHt6tWPm z0eQj3`3$1;@6-LO$G_gYlu>l%eMXc+lHl6Zj<1t2+I@87bJOEUleCb=Gqu5igi>8C zZS5ASZM!5sMp5gUw1$Ss5rz;NIfur?BwB|qMIQeA@v^y#se2x*nRZ4-889_PSZytQ zA=ly>6hXx1SHSkgo+_%VJ5LNDy(ex2zMuBboSiO7;$svbuh5yU_952A7O%-cfo2kU1KYX9eMdPhjo-TfG}|;@Yz$?1 zPmUCucNihW#Y)HM)p5TlNHvuKuNJinVj&dOJ{|q8=F?}Jxe#+n4F^;lW zj{udBS#ZVf3Q9^AAg9Q}1vhN~AnJaUhOU~=+{dLa8RSupJKU7YHagljulnW5*KI?8 z{skP|@J9PosCzv23qF{2rd2^AT}y>%EnkW;hy>XH(4vIH;WB|Tw~)v!3;=8r(yR6ff-;YYA99G^1Ie?$m)B zk6=0fzJ{GgoyVZP@r|ItvV4MIEN!sDG<^#???0bIzjIo7*}}aqb7k<@HLNIeFbgMv z>J^NjIie9`%um0k}#D_wDkc5Al$H4SLgchaKDb2ZwXel*BCjEc~x1yqxS$XPZ{U={-S4YFZQf% zH&%nn5YGj1f-XBQPPOiGjtNwSD|=*UtC;tn3WHNLXJF0**S5O>cYxcO!u@E2 z_?0>-DY;>O=r`MRW4Z|xRii->yZ!j(q7hhu2MJorE1Mz>dq2=+?wJF)Y1zl|k4oLg zO@FBKvPygm-HJ!u+m5U;`HpceX}))zKg^gn(|0?CGdjc&laiZYPDn$|42WL$bJ%gz zo2_)gdk|tzDYq<2C6Mz)h<>I$3*jzdp>}`j=Px)BXW6d%U)(2emo*3>B!_teX=!Ld zc4K)f0LpDZRI11U{lt5IjOW!CuW^P7C8salWz?KH68EU+PlW1GRfGt|Tt)czd`OUS@8NUXA%=s*ha-+_Glape|^t1Z5#ukZ1T=WgjIzm~sgIZH&xs5e-R ztY)Ogn7f5-Pm90S|E~6P6oF~0!4%DiF z=*;uX{jjc6(D&d!C@9sFt|}{Y3z%*+XH@5a^X1RxD7eGRE6m+ndHZcOE0As6>NJ^l~`4mo~nOM~6R z;$!iA52Q?ZFstx2gN$^)?m6CTPBNtfS9-3eLg`!@P9MF^H!a+XFLqdLvzG^D zT_MpGwkTJF*hxYCvF`|adIGCZ$?PbfzSpMvMy=s6{{_uP<3k zgV>h#>|FWIo9o}hnl6*w#;VN0%B8z{;A!7roI^>TJ)grVtmD{zMw;M7rzL00XbhTS zU|FnMjy}!=qj3HDdFS)rW}&)^>MRj@=X(S2+wu4NKl^wm@mHPsv`vvJ3~YOZ)g7;8 zIAv!)3%P<5^0E(TPG`g7Rsa**dNepJWTM_KAxFh|YB>s^W>VHg)T0Wd zjbEkr4pNXp3)ZeRya-7X^FlLnbdK{`t@8{WVR8|%RNNG2TR7)b9!Uk(uI^sv^|_C> zwv}#kYXO@|O4e>*ef-&pDR=s%@st9;+bG(UZ}`DS(nZns%=5t0ZR@b?8hcK2P=IWF zZ0t~}mizKwhplT@UyQiPDfRaC^#K_JFDEDYpt~Cr)%FJ*Pm4fls9GFm2Hy88uQtN9 zYu7T2qKfrUKiB&0dL1@-YV|caYS+D5+sl+yqlEax^rfT>o>ByT4OFHV)P4FWH#^&z zzUnkbr!;N4HwE)Ocrk`0f;l2wS&oB~(*`#+QWkl}l-m>Xi~=IExKd@Z*Y>b8ULWH& z_E6<3qd5sl&9lU_O*SRdcQlKEgpE!ecH@zIiWi&%x#nB}jNxkYz_#k7Ry|(!J z{nI-MzfAUjGLBfZW$)!tzoKmTzDgsd6Tu7t@f`>eDZw&k$aUXjti>WVl{KCh_#a+94-Ebc?T=}-MNHvRM zh0SKQuUzq8Ur*s(;`U6H$Q^S%NHUC(8R6AuQTXvpqCv}HYFAJ|pyasjz1EuZ0xi~2 zs(}t%+|b;7WD|43eWd?>UF6fYoQ_q#l!ItOMh{RS2E;Dr%pdcn*oM2B@$q7T(dufb z^|1~2x%gS)pFhiAuF#+CZ)WG@9P+RojdF_GkDneVR8zQ=fnmcLg7pY>i=hmpZDZMFkXo3|tHKQCp`V4Ops=U=VWI9~$3-@in_T`C=vB^>C%v z7W(*h8pKO6JkDz%cKK_eIz zn{r$bs9VMH;py`y{+FA4=@Xbg&LOp!-tKSBk|M)puLtN7i3&P2s^Vbwmo z=JWhHzwBM6*$wBaJUJMw!>kgSmr=z}&X#)u>;%H_9X;(&GCzo;q42}{pFi_X2R_3n z86Qble)Q$%m()ota@)|I!-h>^M$a}eBhO}0CJy*bhoQb>-;D|Sy`q}Ds)6s9&K#!di`AvP#`=C;*o-co`~NlY zBCy4OupZu}0|%$rg^&^q)XDte;m7-%!+`DsL?FflskB~DAXAPH09!~wNkOl69b1%= zf;qW&U)56Eqp{&G=;2H?I1^R|m$J>WF`jO8e}_t4LmzP)Hk`KF=~?TFkb?;XAZ=7w z4YcI@;@#|Yg94;a>s-;sZ`3MsszDMhtW<2LkSNCRTm;yc!eE*!|@1gBo0E)Izs-C}`0cU9!yyZ9S=1l@Wh`+|l%$M4_&x6 0.5 + pred = pred.astype("int32").compute() + y_test = y_test.compute() + score = accuracy_score_func(y_test, pred) + cv_fold_scores.append(score) + final_score = sum(cv_fold_scores) / len(cv_fold_scores) + return final_score + + +def train_randomforest(trial, *, dataset, client, mode): + params = { + "max_depth": trial.suggest_int("max_depth", 5, 15), + "max_features": trial.suggest_float("max_features", 0.1, 1.0), + "n_estimators": trial.suggest_int("n_estimators", 100, 500, step=10), + "split_criterion": trial.suggest_categorical( + "split_criterion", ["gini", "entropy"] + ), + "min_samples_split": trial.suggest_int("min_samples_split", 2, 1000, log=True), + "n_bins": 256, + } + + cv_fold_scores = [] + for i_fold in range(n_cv_folds): + X_train, y_train, X_test, y_test = preprocess_data( + dataset, client=client, i_fold=i_fold, mode=mode + ) + + if mode == "gpu": + trained_model = RF_gpu(client=client, **params) + accuracy_score_func = accuracy_score_gpu + else: + trained_model = RF_cpu(**params) + accuracy_score_func = accuracy_score_cpu + + trained_model.fit(X_train, y_train) + pred = trained_model.predict(X_test).compute() + y_test = y_test.compute() + score = accuracy_score_func(y_test, pred) + cv_fold_scores.append(score) + final_score = sum(cv_fold_scores) / len(cv_fold_scores) + return final_score + + +def main(args): + tstart = time.perf_counter() + + study = optuna.create_study( + sampler=RandomSampler(seed=args.seed), direction="maximize" + ) + + if args.mode == "gpu": + cluster = LocalCUDACluster() + else: + cluster = LocalCluster(n_workers=os.cpu_count()) + + with Client(cluster) as client: + dataset = ingest_data(mode=args.mode) + client.persist(dataset) + if args.model_type == "XGBoost": + study.optimize( + lambda trial: train_xgboost( + trial, dataset=dataset, client=client, mode=args.mode + ), + n_trials=100, + n_jobs=1, + ) + else: + study.optimize( + lambda trial: train_randomforest( + trial, dataset=dataset, client=client, mode=args.mode + ), + n_trials=100, + n_jobs=1, + ) + + tend = time.perf_counter() + print(f"Time elapsed: {tend - tstart} sec") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model-type", type=str, required=True, choices=["XGBoost", "RandomForest"] + ) + parser.add_argument("--mode", required=True, choices=["gpu", "cpu"]) + parser.add_argument("--seed", required=False, type=int, default=1) + args = parser.parse_args() + main(args) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb new file mode 100644 index 00000000..7e6ef965 --- /dev/null +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb @@ -0,0 +1,268 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a51c95d1-b447-4f1b-9571-cf597ca93ef4", + "metadata": {}, + "source": [ + " **HPO Benchmarking with RAPIDS and Dask** \n", + "\n", + "Hyper-Parameter Optimization (HPO) helps to find the best version of a model by exploring the space of possible configurations. While generally desirable, this search is computationally expensive and time-consuming.\n", + "\n", + "In the notebook demo below, we compare benchmarking results to show how GPU can accelerate HPO tuning jobs relative to CPU.\n", + "\n", + "For instance, we find a x speedup in wall clock time (6 hours vs 3+ days) and a x reduction in cost when comparing between GPU and CPU EC2 instances on 100 XGBoost HPO trials using 10 parallel workers on 10 years of the Airline Dataset.\n", + "\n", + "For more check out our AWS blog(link)." + ] + }, + { + "cell_type": "markdown", + "id": "d0d178f5-fc5d-471e-9898-544e5fdbc271", + "metadata": {}, + "source": [ + " **Preamble** " + ] + }, + { + "cell_type": "markdown", + "id": "8c311bd4-76a1-4ee7-8841-5d44ca052566", + "metadata": {}, + "source": [ + "You can set up local environment but it is recommended to launch a Virtual Machine service (Azure, AWS, GCP, etc).\n", + "\n", + "For the purposes of this notebook, we will be utilizing the [Amazon Machine Image (AMI)](https://aws.amazon.com/releasenotes/aws-deep-learning-ami-gpu-tensorflow-2-12-amazon-linux-2/) as the starting point.\n", + "\n", + "\n", + "````{docref} /cloud/aws/\n", + "Please follow instructions in [AWS Elastic Cloud Compute)](../../cloud/aws/ec2) to launch an EC2 instance with GPUs, the NVIDIA Driver and the NVIDIA Container Runtime.\n", + "\n", + "```{note}\n", + "When configuring your instance ensure you select the [Deep Learning AMI GPU TensorFlow or PyTorch](https://docs.aws.amazon.com/dlami/latest/devguide/appendix-ami-release-notes.html) in the AMI selection box under **\"Amazon Machine Image (AMI)\"**\n", + "\n", + "![](../../_static/images/examples/xgboost-rf-gpu-cpu-benchmark/amazon-deeplearning-ami.png)\n", + "```\n", + "\n", + "Once your instance is running and you have access to Jupyter save this notebook and run through the cells.\n", + "\n", + "````\n" + ] + }, + { + "cell_type": "markdown", + "id": "84b3fbaf-754b-45a6-959c-7163edfe5c4f", + "metadata": {}, + "source": [ + " 2.1 - Dataset \n", + "\n", + "The data source for this workflow is 3 years of the [Airline On-Time Statistics](https://www.transtats.bts.gov/ONTIME/) dataset from the US Bureau of Transportation.\n", + "\n", + "The public dataset contains logs/features about flights in the United States (17 airlines) including:\n", + "\n", + "* Locations and distance ( Origin, Dest, Distance )\n", + "* Airline / carrier ( Reporting_Airline )\n", + "* Scheduled departure and arrival times ( CRSDepTime and CRSArrTime )\n", + "* Actual departure and arrival times ( DpTime and ArrTime )\n", + "* Difference between scheduled & actual times ( ArrDelay and DepDelay )\n", + "* Binary encoded version of late, aka our target variable ( ArrDelay15 )\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "727b5a5b-87cf-4ff6-84f2-4485c7c0470e", + "metadata": {}, + "outputs": [], + "source": [ + "# !aws configure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3bd10f79-23b2-440e-89b4-f6e4a5c9f3bb", + "metadata": {}, + "outputs": [], + "source": [ + "## DOWNLOAD THE DATASET\n", + "!aws s3 cp --recursive s3://sagemaker-rapids-hpo-us-west-2/3_year/ ./data/" + ] + }, + { + "cell_type": "markdown", + "id": "3b1759de-98af-4628-a79b-a236a2dee5a2", + "metadata": {}, + "source": [ + " 2.2 - Local Cluster " + ] + }, + { + "cell_type": "markdown", + "id": "533be0b1-0d5e-46b3-9ff1-dd71751fe68f", + "metadata": {}, + "source": [ + "To maximize on efficiency, we launch a `LocalCUDACluster` that utilizes GPUs for distributed computing. Then connect a Dask Client to submit and manage computations on the cluster. Refer to this (link) for more information on how to achieve this.\n", + "\n", + "Submit dataset to the Dask client, instructing Dask to store the dataset in memory at all times. This can improve performance by avoiding unnecessary data transfers during the hpo process. \n" + ] + }, + { + "cell_type": "markdown", + "id": "e7d57ba2-df8a-4757-a0df-44b3cd73b75c", + "metadata": {}, + "source": [ + " **ML Workflow** " + ] + }, + { + "cell_type": "markdown", + "id": "20ddb8fd-2ac8-4257-8f7f-47f71c6d76c2", + "metadata": {}, + "source": [ + "In order to work with RAPIDS container, the entrypoint logic should parse arguments, load and split data, build and train a model, score/evaluate the trained model, and emit an output representing the final score for the given hyperparameter setting." + ] + }, + { + "cell_type": "markdown", + "id": "26fcb2c4-526c-466e-88b6-00c202f494c2", + "metadata": {}, + "source": [ + "`Optuna` is a hyperparameter optimization library in Python. We create an Optuna `study object` that provides a framework to define the search space, objective function, and optimization algorith for the hpo process. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a2d4ec4-b6e4-4546-8177-505f2739c0c2", + "metadata": {}, + "outputs": [], + "source": [ + "%cd code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f715efe-0e23-4b12-aac3-49e0e3b575b4", + "metadata": {}, + "outputs": [], + "source": [ + "ls" + ] + }, + { + "cell_type": "markdown", + "id": "a89edfea-ca14-4d26-94c6-0ef8eaf02d77", + "metadata": {}, + "source": [ + " **Build RAPIDS Container** " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9919b0eb-70f6-43cb-8d64-5bc9e3b2ace9", + "metadata": {}, + "outputs": [], + "source": [ + "!nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd07ad98-9d9d-4f9f-ba4e-1b0a1646cc4e", + "metadata": {}, + "outputs": [], + "source": [ + "cat Dockerfile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b72cbf8-1f93-4e63-a767-d87210697eea", + "metadata": {}, + "outputs": [], + "source": [ + "!docker images" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bff0ccfd-dc68-4c1a-8a31-e74a5d02b8a9", + "metadata": {}, + "outputs": [], + "source": [ + "!docker build -t rapids-tco-benchmark:v23.06 ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ab1959fa-f70d-4940-8204-1946acd0e8bb", + "metadata": {}, + "outputs": [], + "source": [ + "!docker images" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a2b8b7f-1634-42b8-81e8-2605195d2a1d", + "metadata": {}, + "outputs": [], + "source": [ + "# !tmux" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00b59cf6-963e-4a08-85f7-267b83a99cb1", + "metadata": {}, + "outputs": [], + "source": [ + "!docker run -it --gpus all -p 8888:8888 -p 8787:8787 -p 8786:8786 -v \\\n", + " /home/ec2-user/tco_hpo_gpu_cpu_perf_benchmark:/rapids/notebooks/host \\\n", + " rapids-tco-benchmark:v23.06 \n" + ] + }, + { + "cell_type": "markdown", + "id": "55f881ee-ce7f-4810-a6b7-fa8aa72d91f3", + "metadata": {}, + "source": [ + " **Run HPO** \n", + "\n", + "Navigate to the host directory inside the container and run the python script with the following command : \n", + "\n", + " python ./hpo.py --model-type \"XGBoost\" --mode \"gpu\" > xgboost_gpu.txt 2>&1\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "rapids-23.06", + "language": "python", + "name": "rapids-23.06" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From ea3550537f6786f197c7d2666e4a25cdcb8124a8 Mon Sep 17 00:00:00 2001 From: skirui-source Date: Thu, 13 Jul 2023 16:46:41 -0700 Subject: [PATCH 02/15] removed unrelated files to the PR --- source/examples/index.md | 1 - .../rapids-azureml-hpo/notebook.ipynb | 541 ------------------ 2 files changed, 542 deletions(-) delete mode 100644 source/examples/rapids-azureml-hpo/notebook.ipynb diff --git a/source/examples/index.md b/source/examples/index.md index 852bdc2d..f92b927b 100644 --- a/source/examples/index.md +++ b/source/examples/index.md @@ -13,5 +13,4 @@ rapids-sagemaker-hpo/notebook rapids-ec2-mnmg/notebook rapids-autoscaling-multi-tenant-kubernetes/notebook xgboost-randomforest-gpu-hpo-dask/notebook -rapids-azureml-hpo/notebook ``` diff --git a/source/examples/rapids-azureml-hpo/notebook.ipynb b/source/examples/rapids-azureml-hpo/notebook.ipynb deleted file mode 100644 index d9e42bf4..00000000 --- a/source/examples/rapids-azureml-hpo/notebook.ipynb +++ /dev/null @@ -1,541 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "tags": [ - "workflows/hpo", - "cloud/azure/ml", - "library/cudf", - "library/cuml", - "library/randomforest" - ] - }, - "source": [ - "# Train and hyperparameter tune with RAPIDS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisites" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create an Azure ML [Workspace](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#workspace) and setup environment on local computer or Azure ML Compute Instance, following these [instructions](https://docs.rapids.ai/deployment/stable/cloud/azure/azureml/#azure-ml-compute-instance).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# verify Azure ML SDK version\n", - "\n", - "%pip show azure-ai-ml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Initialize workspace" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Initialize`MLClient` class to handle the workspace you created in the prerequisites step. \n", - "\n", - "You can manually provide the workspace details or call `MLClient.from_config(credential, path)`\n", - "to create a workspace object from the details stored in `config.json`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.ml import MLClient\n", - "from azure.identity import DefaultAzureCredential\n", - "\n", - "\n", - "# Get a handle to the workspace\n", - "ml_client = MLClient(\n", - " credential=DefaultAzureCredential(),\n", - " subscription_id=\"fc4f4a6b-4041-4b1c-8249-854d68edcf62\",\n", - " resource_group_name=\"rapidsai-deployment\",\n", - " workspace_name=\"rapids-aml-cluster\",\n", - ")\n", - "\n", - "print(\n", - " \"Workspace name: \" + ml_client.workspace_name,\n", - " \"Subscription id: \" + ml_client.subscription_id,\n", - " \"Resource group: \" + ml_client.resource_group_name,\n", - " sep=\"\\n\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "## Access data from Datastore URI" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this example, we will use 20 million rows of the airline dataset. The [datastore uri](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-access-data-interactive?tabs=adls#access-data-from-a-datastore-uri-like-a-filesystem-preview) below references a data storage location (path) containing the parquet files" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "datastore_name = \"workspaceartifactstore\"\n", - "dataset = \"airline_20000000.parquet\"\n", - "\n", - "# Datastore uri format:\n", - "data_uri = f\"azureml://subscriptions/{ml_client.subscription_id}/resourcegroups/{ml_client.resource_group_name}/workspaces/{ml_client.workspace_name}/datastores/{datastore_name}/paths/{dataset}\"\n", - "\n", - "print(\"data uri:\", \"\\n\", data_uri)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create AML compute" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You will need to create an Azure ML managed compute target ([AmlCompute](https://docs.microsoft.com/azure/machine-learning/service/how-to-set-up-training-targets#amlcompute)) for training your model.\n", - "\n", - "This notebook will use 10 nodes for hyperparameter optimization, you can modify `max_instances` based on available quota in the desired region. Similar to other Azure ML services, there are limits on AmlCompute, this [article](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-manage-quotas) includes details on the default limits and how to request more quota." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`size` describes the virtual machine type and size that will be used in the cluster. RAPIDS requires NVIDIA Pascal or newer architecture, so \n", - "you will need to select compute targets from one of the \n", - "[GPU virtual machines in Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes-gpu) provisioned with P40 and V100 GPUs : `NC_v2`, `NC_v3`, `ND` or `ND_v2` \n", - "\n", - "Let's create an `AmlCompute` cluster of `Standard_NC12s_v3` GPU VMs:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.ml.entities import AmlCompute\n", - "\n", - "# specify aml compute name.\n", - "gpu_compute_target = \"rapids-cluster\"\n", - "\n", - "try:\n", - " # let's see if the compute target already exists\n", - " gpu_target = ml_client.compute.get(gpu_compute_target)\n", - " print(f\"found compute target. Will use {gpu_compute_target}\")\n", - "except:\n", - " print(\"Creating a new gpu compute target...\")\n", - "\n", - " gpu_target = AmlCompute(\n", - " name=\"rapids-cluster\",\n", - " type=\"amlcompute\",\n", - " size=\"STANDARD_NC12S_V3\",\n", - " max_instances=5,\n", - " idle_time_before_scale_down=300,\n", - " )\n", - " ml_client.compute.begin_create_or_update(gpu_target).result()\n", - "\n", - " print(\n", - " f\"AMLCompute with name {gpu_target.name} is created, the compute size is {gpu_target.size}\"\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "## Prepare training script" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [ - "library/cuml" - ] - }, - "source": [ - "Create a project directory with your code to run on the remote resource. This includes the training script and additional files your training script depends on. In this example, the training script is provided:\n", - "\n", - "`train_rapids.py`- entry script for RAPIDS Environment, includes loading dataset into cuDF dataframe, training with Random Forest and inference using cuML." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "project_folder = \"./train_rapids\" # create folder in same dir\n", - "os.makedirs(project_folder, exist_ok=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will log some parameters and metrics including highest accuracy, using mlflow within the training script:\n", - "\n", - "```console\n", - "import mlflow\n", - "\n", - "mlflow.log_metric('Accuracy', np.float(global_best_test_accuracy))\n", - "```\n", - "\n", - "These run metrics will become particularly important when we begin hyperparameter tuning our model in the 'Tune model hyperparameters' section." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Copy the training script `train_rapids.py` into your project directory:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import shutil\n", - "\n", - "\n", - "notebook_path = os.path.realpath(\n", - " \"__file__\" + \"/../../code\"\n", - ") # dir containing the training scrips\n", - "rapids_script = os.path.join(notebook_path, \"train_rapids.py\")\n", - "azure_script = os.path.join(notebook_path, \"rapids_csp_azure.py\")\n", - "\n", - "\n", - "shutil.copy(rapids_script, project_folder)\n", - "shutil.copy(azure_script, project_folder)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "## Train model on the remote compute" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that you have your data and training script prepared, you are ready to train on your remote compute:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create experiment\n", - "\n", - "Track all the runs in your workspace" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "experiment_name = \"test_rapids_gpu_cluster\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Setup Environment" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We'll be using a [custom](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-manage-environments-v2?tabs=python#create-an-environment-from-a-docker-image) RAPIDS docker image to setup the environment. This is available in [rapidsai/rapidsai repo](https://hub.docker.com/r/rapidsai/rapidsai/) on DockerHub." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# RUN THIS CODE ONCE TO SETUP ENVIRONMENT\n", - "from azure.ai.ml.entities import Environment, BuildContext\n", - "\n", - "env_docker_image = Environment(\n", - " build=BuildContext(path=\"./docker\"),\n", - " name=\"rapids-mlflow\",\n", - " description=\"RAPIDS environment with azureml-mlflow\",\n", - ")\n", - "\n", - "ml_client.environments.create_or_update(env_docker_image)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Submit the training job " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will configure and run a training job using the`command`class. The [command](https://learn.microsoft.com/en-us/python/api/azure-ai-ml/azure.ai.ml?view=azure-python#azure-ai-ml-command) can be used to run standalone jobs or as a function inside pipelines.\n", - "`inputs` is a dictionary of command-line arguments to pass to the training script.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "library/randomforest", - "library/cudf" - ] - }, - "outputs": [], - "source": [ - "from azure.ai.ml import command, Input\n", - "\n", - "\n", - "command_job = command(\n", - " environment=\"rapids-mlflow:1\",\n", - " experiment_name=experiment_name,\n", - " code=project_folder,\n", - " command=\"python train_rapids.py --data_dir ${{inputs.data_dir}} --n_bins ${{inputs.n_bins}} --compute ${{inputs.compute}} --cv_folds ${{inputs.cv_folds}}\\\n", - " --n_estimators ${{inputs.n_estimators}} --max_depth ${{inputs.max_depth}} --max_features ${{inputs.max_features}}\",\n", - " inputs={\n", - " \"data_dir\": Input(type=\"uri_file\", path=data_uri),\n", - " \"n_bins\": 32,\n", - " \"compute\": \"single-GPU\", # multi-GPU for algorithms via Dask\n", - " \"cv_folds\": 5,\n", - " \"n_estimators\": 100,\n", - " \"max_depth\": 6,\n", - " \"max_features\": 0.3,\n", - " },\n", - " compute=\"rapids-cluster\",\n", - ")\n", - "\n", - "\n", - "# submit the command\n", - "returned_job = ml_client.jobs.create_or_update(command_job)\n", - "\n", - "# get a URL for the status of the job\n", - "returned_job.studio_url" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tune model hyperparameters" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can optimize our model's hyperparameters and improve the accuracy using Azure Machine Learning's hyperparameter tuning capabilities." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Start a hyperparameter sweep" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's define the hyperparameter space to sweep over. We will tune `n_estimators`, `max_depth` and `max_features` parameters. In this example we will use random sampling to try different configuration sets of hyperparameters and maximize `Accuracy`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.ml.sweep import Choice, Uniform, MedianStoppingPolicy\n", - "\n", - "command_job_for_sweep = command_job(\n", - " n_estimators=Choice(values=range(50, 500)),\n", - " max_depth=Choice(values=range(5, 19)),\n", - " max_features=Uniform(min_value=0.2, max_value=1.0),\n", - ")\n", - "\n", - "# apply sweep parameter to obtain the sweep_job\n", - "sweep_job = command_job_for_sweep.sweep(\n", - " compute=\"rapids-cluster\",\n", - " sampling_algorithm=\"random\",\n", - " primary_metric=\"Accuracy\",\n", - " goal=\"Maximize\",\n", - ")\n", - "\n", - "\n", - "# Define the limits for this sweep\n", - "sweep_job.set_limits(\n", - " max_total_trials=5, max_concurrent_trials=2, timeout=18000, trial_timeout=3600\n", - ")\n", - "\n", - "\n", - "# Specify your experiment details\n", - "sweep_job.display_name = \"RF-rapids-sweep-job\"\n", - "sweep_job.description = \"Run RAPIDS hyperparameter sweep job\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This will launch the RAPIDS training script with parameters that were specified in the cell above." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# submit the hpo job\n", - "returned_sweep_job = ml_client.create_or_update(sweep_job)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Monitor SweepJobs runs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "aml_url = returned_sweep_job.studio_url\n", - "\n", - "print(\"Monitor your job at\", aml_url)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Find and register best model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Download the best trial model output" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ml_client.jobs.download(returned_sweep_job.name, output_name=\"model\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Delete cluster" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ml_client.compute.begin_delete(gpu_compute_target.name).wait()" - ] - } - ], - "metadata": { - "kernel_info": { - "name": "rapids" - }, - "kernelspec": { - "display_name": "rapids", - "language": "python", - "name": "rapids" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - }, - "microsoft": { - "ms_spell_check": { - "ms_spell_check_language": "en" - } - }, - "nteract": { - "version": "nteract-front-end@1.0.0" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} From 5ff363061fefd3085bb5532d6b861361c9873655 Mon Sep 17 00:00:00 2001 From: skirui-source Date: Fri, 14 Jul 2023 11:49:02 -0700 Subject: [PATCH 03/15] fix name of criterion param in rf_cpu --- source/examples/xgboost-rf-gpu-cpu-benchmark/hpo.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/hpo.py b/source/examples/xgboost-rf-gpu-cpu-benchmark/hpo.py index 5e11ce16..139b7b6b 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/hpo.py +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/hpo.py @@ -123,9 +123,7 @@ def train_randomforest(trial, *, dataset, client, mode): "max_depth": trial.suggest_int("max_depth", 5, 15), "max_features": trial.suggest_float("max_features", 0.1, 1.0), "n_estimators": trial.suggest_int("n_estimators", 100, 500, step=10), - "split_criterion": trial.suggest_categorical( - "split_criterion", ["gini", "entropy"] - ), + "criterion": trial.suggest_categorical("criterion", ["gini", "entropy"]), "min_samples_split": trial.suggest_int("min_samples_split", 2, 1000, log=True), "n_bins": 256, } From 26e14378e82e567182909507c48a4e7fdd185f3b Mon Sep 17 00:00:00 2001 From: skirui-source Date: Mon, 17 Jul 2023 11:09:15 -0700 Subject: [PATCH 04/15] few edits --- source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb index 7e6ef965..83ea20d6 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb @@ -13,7 +13,7 @@ "\n", "For instance, we find a x speedup in wall clock time (6 hours vs 3+ days) and a x reduction in cost when comparing between GPU and CPU EC2 instances on 100 XGBoost HPO trials using 10 parallel workers on 10 years of the Airline Dataset.\n", "\n", - "For more check out our AWS blog(link)." + "For more check out our blog(link)." ] }, { @@ -102,9 +102,9 @@ "id": "533be0b1-0d5e-46b3-9ff1-dd71751fe68f", "metadata": {}, "source": [ - "To maximize on efficiency, we launch a `LocalCUDACluster` that utilizes GPUs for distributed computing. Then connect a Dask Client to submit and manage computations on the cluster. Refer to this (link) for more information on how to achieve this.\n", + "To maximize on efficiency, we launch either a `LocalCluster` for cpu or `LocalCUDACluster` that utilizes GPUs for distributed computing. Then connect a Dask Client to submit and manage computations on the cluster. \n", "\n", - "Submit dataset to the Dask client, instructing Dask to store the dataset in memory at all times. This can improve performance by avoiding unnecessary data transfers during the hpo process. \n" + "Submit and \"persist\" the dataset to the Dask client, which instructs Dask to store the dataset in memory at all times for faster performance by avoiding unnecessary data transfers during the hpo process. \n" ] }, { From 1cecaa2013bd17f7bf56f75f5dde7080fbcaeedb Mon Sep 17 00:00:00 2001 From: skirui-source Date: Mon, 17 Jul 2023 14:27:53 -0700 Subject: [PATCH 05/15] add new subsections in ML Workflow --- .../notebook.ipynb | 81 +++++++++---------- 1 file changed, 39 insertions(+), 42 deletions(-) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb index 83ea20d6..a08067b3 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb @@ -11,9 +11,7 @@ "\n", "In the notebook demo below, we compare benchmarking results to show how GPU can accelerate HPO tuning jobs relative to CPU.\n", "\n", - "For instance, we find a x speedup in wall clock time (6 hours vs 3+ days) and a x reduction in cost when comparing between GPU and CPU EC2 instances on 100 XGBoost HPO trials using 10 parallel workers on 10 years of the Airline Dataset.\n", - "\n", - "For more check out our blog(link)." + "For instance, we find a x speedup in wall clock time and a x reduction in cost when comparing between GPU and CPU EC2 instances on 100 XGBoost HPO trials on 3 years of the Airline Dataset.\n" ] }, { @@ -48,16 +46,26 @@ "````\n" ] }, + { + "cell_type": "markdown", + "id": "e7d57ba2-df8a-4757-a0df-44b3cd73b75c", + "metadata": {}, + "source": [ + " **ML Workflow** \n" + ] + }, { "cell_type": "markdown", "id": "84b3fbaf-754b-45a6-959c-7163edfe5c4f", "metadata": {}, "source": [ - " 2.1 - Dataset \n", + " Dataset \n", + "\n", + "We leverage the `Airline` dataset, which is a large public tracker of US domestic flight logs which we offer in various sizes (1 year, 3 year, and 10 year) and in [Parquet](https://parquet.apache.org/) (compressed column storage) format. The machine learning objective with this dataset is to predict whether flights will be more than 15 minutes late arriving to their destination.\n", "\n", - "The data source for this workflow is 3 years of the [Airline On-Time Statistics](https://www.transtats.bts.gov/ONTIME/) dataset from the US Bureau of Transportation.\n", + "We host the demo dataset in public S3 demo buckets in both the `us-east-1` or `us-west-2`. To optimize performance, we recommend that you access the s3 bucket in the same region as your EC2 instance to reduce network latency and data transfer costs. \n", "\n", - "The public dataset contains logs/features about flights in the United States (17 airlines) including:\n", + "For this demo, we are using the 3_year dataset, which includes the following features to mention a few:\n", "\n", "* Locations and distance ( Origin, Dest, Distance )\n", "* Airline / carrier ( Reporting_Airline )\n", @@ -75,7 +83,8 @@ "metadata": {}, "outputs": [], "source": [ - "# !aws configure" + "# configure aws credentials for access to S3 storage\n", + "!aws configure" ] }, { @@ -85,70 +94,58 @@ "metadata": {}, "outputs": [], "source": [ - "## DOWNLOAD THE DATASET\n", + "# download dataset from S3 bucket to your current working dir\n", "!aws s3 cp --recursive s3://sagemaker-rapids-hpo-us-west-2/3_year/ ./data/" ] }, { "cell_type": "markdown", - "id": "3b1759de-98af-4628-a79b-a236a2dee5a2", + "id": "a4bc101b-3206-4f86-9b0d-d4313ad1ee43", "metadata": {}, "source": [ - " 2.2 - Local Cluster " + " Algorithm \n", + "\n", + "From a ML/algorithm perspective, we offer `XGBoost` and `RandomForest`. You are free to switch between these algorithm choices and everything in the example will continue to work.\n", + "\n", + "We can also optionally increase robustness via reshuffles of the train-test split (i.e., cross-validation folds). Typical values here are between 3 and 10 folds but we will use `n_cv_folds = 5`" ] }, { "cell_type": "markdown", - "id": "533be0b1-0d5e-46b3-9ff1-dd71751fe68f", + "id": "feef53b4-d7e0-43e2-b85b-372bd2d882f7", "metadata": {}, "source": [ - "To maximize on efficiency, we launch either a `LocalCluster` for cpu or `LocalCUDACluster` that utilizes GPUs for distributed computing. Then connect a Dask Client to submit and manage computations on the cluster. \n", + " Search Range \n", "\n", - "Submit and \"persist\" the dataset to the Dask client, which instructs Dask to store the dataset in memory at all times for faster performance by avoiding unnecessary data transfers during the hpo process. \n" + "In order to work with RAPIDS container, the entrypoint logic should parse arguments, load and split data, build and train a model, score/evaluate the trained model, and emit an output representing the final score for the given hyperparameter setting.\n", + "\n", + "`Optuna` is a hyperparameter optimization library in Python. We create an Optuna study object that provides a framework to define the search space, objective function, and optimization algorithm for the hpo process.\n" ] }, { - "cell_type": "markdown", - "id": "e7d57ba2-df8a-4757-a0df-44b3cd73b75c", + "cell_type": "code", + "execution_count": null, + "id": "c2ac0c4b-5581-4bd9-ae4e-e4aedef30ac8", "metadata": {}, - "source": [ - " **ML Workflow** " - ] + "outputs": [], + "source": [] }, { "cell_type": "markdown", - "id": "20ddb8fd-2ac8-4257-8f7f-47f71c6d76c2", + "id": "3b1759de-98af-4628-a79b-a236a2dee5a2", "metadata": {}, "source": [ - "In order to work with RAPIDS container, the entrypoint logic should parse arguments, load and split data, build and train a model, score/evaluate the trained model, and emit an output representing the final score for the given hyperparameter setting." + " Local Cluster " ] }, { "cell_type": "markdown", - "id": "26fcb2c4-526c-466e-88b6-00c202f494c2", - "metadata": {}, - "source": [ - "`Optuna` is a hyperparameter optimization library in Python. We create an Optuna `study object` that provides a framework to define the search space, objective function, and optimization algorith for the hpo process. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6a2d4ec4-b6e4-4546-8177-505f2739c0c2", - "metadata": {}, - "outputs": [], - "source": [ - "%cd code" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5f715efe-0e23-4b12-aac3-49e0e3b575b4", + "id": "533be0b1-0d5e-46b3-9ff1-dd71751fe68f", "metadata": {}, - "outputs": [], "source": [ - "ls" + "To maximize on efficiency, we launch either a `LocalCluster` for cpu or `LocalCUDACluster` that utilizes GPUs for distributed computing. Then connect a Dask Client to submit and manage computations on the cluster. \n", + "\n", + "WE can then submit and \"persist\" the dataset to the Dask client, instructing Dask to store the dataset in memory at all times for faster performance.\n" ] }, { From d3e0af571068858ccfe6dfba7ecce37de17d65bb Mon Sep 17 00:00:00 2001 From: skirui-source Date: Tue, 18 Jul 2023 01:40:29 -0700 Subject: [PATCH 06/15] added more details for Ml workflow, optuna optimization --- .../xgboost-rf-gpu-cpu-benchmark/Dockerfile | 6 +- .../notebook.ipynb | 105 ++++++++++++++---- 2 files changed, 86 insertions(+), 25 deletions(-) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/Dockerfile b/source/examples/xgboost-rf-gpu-cpu-benchmark/Dockerfile index f1996ded..60c72b4f 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/Dockerfile +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/Dockerfile @@ -1,7 +1,5 @@ -# FROM nvcr.io/nvidia/rapidsai/rapidsai-core:23.06-cuda11.5-runtime-ubuntu20.04-py3.10 +#FROM rapidsai/rapidsai:23.06-cuda11.5-runtime-ubuntu20.04-py3.10 -FROM rapidsai/rapidsai:23.06-cuda11.5-runtime-ubuntu20.04-py3.10 - -#FROM rapidsai/rapidsai:23.06-cuda11.8-runtime-ubuntu22.04-py3.10 +FROM rapidsai/rapidsai:23.06-cuda11.8-runtime-ubuntu22.04-py3.10 RUN mamba install -y -n rapids optuna diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb index a08067b3..df7fe7a9 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb @@ -51,7 +51,11 @@ "id": "e7d57ba2-df8a-4757-a0df-44b3cd73b75c", "metadata": {}, "source": [ - " **ML Workflow** \n" + " **ML Workflow** \n", + "\n", + "In order to work with RAPIDS container, the entrypoint logic should parse arguments, load and split data, build and train a model, score/evaluate the trained model, and emit an output representing the final score for the given hyperparameter setting.\n", + "\n", + "Let's have a step-by-step look at each stage of the ML workflow:" ] }, { @@ -107,7 +111,17 @@ "\n", "From a ML/algorithm perspective, we offer `XGBoost` and `RandomForest`. You are free to switch between these algorithm choices and everything in the example will continue to work.\n", "\n", - "We can also optionally increase robustness via reshuffles of the train-test split (i.e., cross-validation folds). Typical values here are between 3 and 10 folds but we will use `n_cv_folds = 5`" + "```python\n", + "parser = argparse.ArgumentParser()\n", + "parser.add_argument(\n", + " \"--model-type\", type=str, required=True, choices=[\"XGBoost\", \"RandomForest\"]\n", + ")\n", + "```\n", + "\n", + "We can also optionally increase robustness via reshuffles of the train-test split (i.e., cross-validation folds). Typical values are between 3 and 10 folds. We will use \n", + "```python\n", + "n_cv_folds = 5\n", + "```\n" ] }, { @@ -117,25 +131,41 @@ "source": [ " Search Range \n", "\n", - "In order to work with RAPIDS container, the entrypoint logic should parse arguments, load and split data, build and train a model, score/evaluate the trained model, and emit an output representing the final score for the given hyperparameter setting.\n", + "One of the most important choices when running HPO is to choose the bounds of the hyperparameter search process. In this notebook, we leverage the power of `Optuna`, a widely used Python library for hyperparameter optimization as such:\n", + "\n", + "\n", + "1) Define the Objective Function. Represent the model training and evaluation process, takes hyperparameters as inputs and returns a metric to optimize (e.g., accuracy, loss). Refer to `train_xgboost()` and `train_randomforest()` in `hpo.py`\n", + "\n", + "2. Specify the search space for your hyperparameters using \n", + "\n", + "3. Create an Optuna study object to keep track of trials and their corresponding hyperparameter configurations and evaluation metrics.\n", + "\n", + "```python\n", + "study = optuna.create_study(\n", + " sampler=RandomSampler(seed=args.seed), direction=\"maximize\"\n", + " )\n", + "```\n", "\n", - "`Optuna` is a hyperparameter optimization library in Python. We create an Optuna study object that provides a framework to define the search space, objective function, and optimization algorithm for the hpo process.\n" + "4. Select an optimization algorithm to determine how Optuna explores and exploits the search space to find optimal configurations. As shown in the code above, \n", + "\n", + "5. Run the Optimization by calling the Optuna's `optimize()` function on the study object. You can specify the number of trials or number of parallel jobs to run.\n", + "\n", + "```python\n", + " study.optimize(lambda trial: train_xgboost(\n", + " trial, dataset=dataset, client=client, mode=args.mode\n", + " ),\n", + " n_trials=100,\n", + " n_jobs=1,\n", + " )\n", + "```" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "c2ac0c4b-5581-4bd9-ae4e-e4aedef30ac8", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "id": "3b1759de-98af-4628-a79b-a236a2dee5a2", "metadata": {}, "source": [ - " Local Cluster " + " Dask Cluster " ] }, { @@ -143,9 +173,20 @@ "id": "533be0b1-0d5e-46b3-9ff1-dd71751fe68f", "metadata": {}, "source": [ - "To maximize on efficiency, we launch either a `LocalCluster` for cpu or `LocalCUDACluster` that utilizes GPUs for distributed computing. Then connect a Dask Client to submit and manage computations on the cluster. \n", + "To maximize on efficiency, we launch a Dask `LocalCluster` for cpu or `LocalCUDACluster` that utilizes GPUs for distributed computing. Then connect a Dask Client to submit and manage computations on the cluster. \n", + "\n", + "We can then ingest the data, and \"persist\" it in memory using dask as follows:\n", "\n", - "WE can then submit and \"persist\" the dataset to the Dask client, instructing Dask to store the dataset in memory at all times for faster performance.\n" + "```python\n", + "if args.mode == \"gpu\":\n", + " cluster = LocalCUDACluster()\n", + "else: # mode == \"cpu\"\n", + " cluster = LocalCluster(n_workers=os.cpu_count())\n", + "\n", + "with Client(cluster) as client:\n", + " dataset = ingest_data(mode=args.mode)\n", + " client.persist(dataset)\n", + "```\n" ] }, { @@ -153,7 +194,21 @@ "id": "a89edfea-ca14-4d26-94c6-0ef8eaf02d77", "metadata": {}, "source": [ - " **Build RAPIDS Container** " + " **Build RAPIDS Container** \n", + "\n", + "Now that we have a fundamental understanding of our workflow process, we can test the code in custom docker container. \n", + "\n", + "Starting with latest rapids docker image, we only need to install `optuna` as the container comes with most necessary packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b72cbf8-1f93-4e63-a767-d87210697eea", + "metadata": {}, + "outputs": [], + "source": [ + "!docker images" ] }, { @@ -177,13 +232,11 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "9b72cbf8-1f93-4e63-a767-d87210697eea", + "cell_type": "markdown", + "id": "a825e0de-a1a1-4c2b-82bb-b33dfc494fd1", "metadata": {}, - "outputs": [], "source": [ - "!docker images" + "Be sure to build and tag appropriately" ] }, { @@ -206,6 +259,16 @@ "!docker images" ] }, + { + "cell_type": "markdown", + "id": "baca52e2-09e7-42f3-bc98-5ee38f9e274f", + "metadata": {}, + "source": [ + "a tedius lengthy process, use a tool like tmux to handle SSH disconnection, avoid hpo runs interruptions \n", + "\n", + "also while running the container, be sure to expose all gpus (why?) and jupyter lab via ports ..." + ] + }, { "cell_type": "code", "execution_count": null, From f48c1f976eee34d114c44e958ac805ef08627845 Mon Sep 17 00:00:00 2001 From: skirui-source Date: Tue, 18 Jul 2023 09:28:51 -0700 Subject: [PATCH 07/15] add more details / description --- .../notebook.ipynb | 121 +++++++++++------- 1 file changed, 73 insertions(+), 48 deletions(-) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb index df7fe7a9..c3abae86 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb @@ -53,7 +53,7 @@ "source": [ " **ML Workflow** \n", "\n", - "In order to work with RAPIDS container, the entrypoint logic should parse arguments, load and split data, build and train a model, score/evaluate the trained model, and emit an output representing the final score for the given hyperparameter setting.\n", + "In order to work with RAPIDS container, the entrypoint logic should parse arguments, load, preprocess and split data, build and train a model, score/evaluate the trained model, and emit an output representing the final score for the given hyperparameter setting.\n", "\n", "Let's have a step-by-step look at each stage of the ML workflow:" ] @@ -69,12 +69,11 @@ "\n", "We host the demo dataset in public S3 demo buckets in both the `us-east-1` or `us-west-2`. To optimize performance, we recommend that you access the s3 bucket in the same region as your EC2 instance to reduce network latency and data transfer costs. \n", "\n", - "For this demo, we are using the 3_year dataset, which includes the following features to mention a few:\n", + "For this demo, we are using the **`3_year`** dataset, which includes the following features to mention a few:\n", "\n", - "* Locations and distance ( Origin, Dest, Distance )\n", - "* Airline / carrier ( Reporting_Airline )\n", - "* Scheduled departure and arrival times ( CRSDepTime and CRSArrTime )\n", - "* Actual departure and arrival times ( DpTime and ArrTime )\n", + "* Date and distance ( Year, Month, Distance )\n", + "* Airline / carrier ( Flight_Number_Reporting_Airline )\n", + "* Actual departure and arrival times ( DepTime and ArrTime )\n", "* Difference between scheduled & actual times ( ArrDelay and DepDelay )\n", "* Binary encoded version of late, aka our target variable ( ArrDelay15 )\n", "\n" @@ -124,6 +123,35 @@ "```\n" ] }, + { + "cell_type": "markdown", + "id": "3b1759de-98af-4628-a79b-a236a2dee5a2", + "metadata": {}, + "source": [ + " Dask Cluster " + ] + }, + { + "cell_type": "markdown", + "id": "533be0b1-0d5e-46b3-9ff1-dd71751fe68f", + "metadata": {}, + "source": [ + "To maximize on efficiency, we launch a Dask `LocalCluster` for cpu or `LocalCUDACluster` that utilizes GPUs for distributed computing. Then connect a Dask Client to submit and manage computations on the cluster. \n", + "\n", + "We can then ingest the data, and \"persist\" it in memory using dask as follows:\n", + "\n", + "```python\n", + "if args.mode == \"gpu\":\n", + " cluster = LocalCUDACluster()\n", + "else: # mode == \"cpu\"\n", + " cluster = LocalCluster(n_workers=os.cpu_count())\n", + "\n", + "with Client(cluster) as client:\n", + " dataset = ingest_data(mode=args.mode)\n", + " client.persist(dataset)\n", + "```\n" + ] + }, { "cell_type": "markdown", "id": "feef53b4-d7e0-43e2-b85b-372bd2d882f7", @@ -131,12 +159,21 @@ "source": [ " Search Range \n", "\n", - "One of the most important choices when running HPO is to choose the bounds of the hyperparameter search process. In this notebook, we leverage the power of `Optuna`, a widely used Python library for hyperparameter optimization as such:\n", + "One of the most important choices when running HPO is to choose the bounds of the hyperparameter search process. In this notebook, we leverage the power of `Optuna`, a widely used Python library for hyperparameter optimization.\n", "\n", + "Here's the quick steps on getting started with Optuna:\n", "\n", - "1) Define the Objective Function. Represent the model training and evaluation process, takes hyperparameters as inputs and returns a metric to optimize (e.g., accuracy, loss). Refer to `train_xgboost()` and `train_randomforest()` in `hpo.py`\n", "\n", - "2. Specify the search space for your hyperparameters using \n", + "1) Define the Objective Function, which represents the model training and evaluation process. It takes hyperparameters as inputs and returns a metric to optimize (e.g, accuracy in our case,). Refer to `train_xgboost()` and `train_randomforest()` in `hpo.py`\n", + "\n", + "2. Specify the search space using the `Trial` object's methods to define the hyperparameters and their corresponding value ranges or distributions. For example:\n", + "\n", + "```python\n", + "\"max_depth\": trial.suggest_int(\"max_depth\", 4, 8),\n", + "\"max_features\": trial.suggest_float(\"max_features\", 0.1, 1.0),\n", + "\"learning_rate\": trial.suggest_float(\"learning_rate\", 0.001, 0.1, log=True),\n", + "\"min_samples_split\": trial.suggest_int(\"min_samples_split\", 2, 1000, log=True),\n", + "```\n", "\n", "3. Create an Optuna study object to keep track of trials and their corresponding hyperparameter configurations and evaluation metrics.\n", "\n", @@ -146,7 +183,7 @@ " )\n", "```\n", "\n", - "4. Select an optimization algorithm to determine how Optuna explores and exploits the search space to find optimal configurations. As shown in the code above, \n", + "4. Select an optimization algorithm to determine how Optuna explores and exploits the search space to find optimal configurations. For instance, the `RandomSampler` is an algorithm provided by the Optuna library that samples hyperparameter configurations randomly from the search space.\n", "\n", "5. Run the Optimization by calling the Optuna's `optimize()` function on the study object. You can specify the number of trials or number of parallel jobs to run.\n", "\n", @@ -160,35 +197,6 @@ "```" ] }, - { - "cell_type": "markdown", - "id": "3b1759de-98af-4628-a79b-a236a2dee5a2", - "metadata": {}, - "source": [ - " Dask Cluster " - ] - }, - { - "cell_type": "markdown", - "id": "533be0b1-0d5e-46b3-9ff1-dd71751fe68f", - "metadata": {}, - "source": [ - "To maximize on efficiency, we launch a Dask `LocalCluster` for cpu or `LocalCUDACluster` that utilizes GPUs for distributed computing. Then connect a Dask Client to submit and manage computations on the cluster. \n", - "\n", - "We can then ingest the data, and \"persist\" it in memory using dask as follows:\n", - "\n", - "```python\n", - "if args.mode == \"gpu\":\n", - " cluster = LocalCUDACluster()\n", - "else: # mode == \"cpu\"\n", - " cluster = LocalCluster(n_workers=os.cpu_count())\n", - "\n", - "with Client(cluster) as client:\n", - " dataset = ingest_data(mode=args.mode)\n", - " client.persist(dataset)\n", - "```\n" - ] - }, { "cell_type": "markdown", "id": "a89edfea-ca14-4d26-94c6-0ef8eaf02d77", @@ -196,9 +204,7 @@ "source": [ " **Build RAPIDS Container** \n", "\n", - "Now that we have a fundamental understanding of our workflow process, we can test the code in custom docker container. \n", - "\n", - "Starting with latest rapids docker image, we only need to install `optuna` as the container comes with most necessary packages" + "Now that we have a fundamental understanding of our workflow process, we can test the code. Starting with latest rapids docker image, we only need to install `optuna` as the container comes with most necessary packages." ] }, { @@ -218,6 +224,7 @@ "metadata": {}, "outputs": [], "source": [ + "# make sure you have the correct CUDAtoolkit version to build latest rapids container\n", "!nvidia-smi" ] }, @@ -236,7 +243,7 @@ "id": "a825e0de-a1a1-4c2b-82bb-b33dfc494fd1", "metadata": {}, "source": [ - "Be sure to build and tag appropriately" + "The build step will be dominated by the download of the RAPIDS image (base layer). If it's already been downloaded the build will take less than 1 minute." ] }, { @@ -264,9 +271,9 @@ "id": "baca52e2-09e7-42f3-bc98-5ee38f9e274f", "metadata": {}, "source": [ - "a tedius lengthy process, use a tool like tmux to handle SSH disconnection, avoid hpo runs interruptions \n", + "Executing benchmark tests can be an arduous and time-consuming procedure that may extend over multiple days.\n", "\n", - "also while running the container, be sure to expose all gpus (why?) and jupyter lab via ports ..." + "By using a tool like [tmux](https://www.redhat.com/sysadmin/introduction-tmux-linux), you can maintain active terminal sessions, ensuring that your tasks continue running even if the SSH connection is interrupted. This allows you to resume your work seamlessly, without losing any progress or requiring you to restart the entire process." ] }, { @@ -276,7 +283,21 @@ "metadata": {}, "outputs": [], "source": [ - "# !tmux" + "# start a tmux session using this command\n", + "\n", + "!tmux" + ] + }, + { + "cell_type": "markdown", + "id": "77df8ce3-39b8-41d9-a538-ae404be25b45", + "metadata": {}, + "source": [ + "When starting the container, be sure to expose the `--gpus all` flag to make all available GPUs on the host machine accessible to the containers within the Docker environment. \n", + "\n", + "Use `-v` (or `--volume`) option to mount working dir from the host machine into the docker container. This enables data or directories on the host machine to be accessible within the container, and any changes made to the mounted files or directories will be reflected in both the host and container environments.\n", + "\n", + "Optional to expose jupyter via ports 8786-8888." ] }, { @@ -298,9 +319,13 @@ "source": [ " **Run HPO** \n", "\n", - "Navigate to the host directory inside the container and run the python script with the following command : \n", + "Navigate to the host directory inside the container and run the python training script with the following command : \n", + "\n", + "```python\n", + "python ./hpo.py --model-type \"XGBoost\" --mode \"gpu\" > xgboost_gpu.txt 2>&1\n", + "```\n", "\n", - " python ./hpo.py --model-type \"XGBoost\" --mode \"gpu\" > xgboost_gpu.txt 2>&1\n" + "The code above will run XGBoost HPO jobs on the gpu and output the benchmark results to a text file. You can run the same for RandomForest by simple chaning `--model type` arg and chnage mode to `cpu` accordingly." ] } ], From 9d5b42c4fdfd883de1fa2b9f22c531a86805a87f Mon Sep 17 00:00:00 2001 From: skirui-source Date: Wed, 2 Aug 2023 02:32:21 -0700 Subject: [PATCH 08/15] fix markdown format issues --- .../notebook.ipynb | 92 +++++++------------ 1 file changed, 31 insertions(+), 61 deletions(-) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb index c3abae86..6d48c168 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb @@ -51,7 +51,7 @@ "id": "e7d57ba2-df8a-4757-a0df-44b3cd73b75c", "metadata": {}, "source": [ - " **ML Workflow** \n", + " **Python ML Workflow** \n", "\n", "In order to work with RAPIDS container, the entrypoint logic should parse arguments, load, preprocess and split data, build and train a model, score/evaluate the trained model, and emit an output representing the final score for the given hyperparameter setting.\n", "\n", @@ -204,38 +204,23 @@ "source": [ " **Build RAPIDS Container** \n", "\n", - "Now that we have a fundamental understanding of our workflow process, we can test the code. Starting with latest rapids docker image, we only need to install `optuna` as the container comes with most necessary packages." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9b72cbf8-1f93-4e63-a767-d87210697eea", - "metadata": {}, - "outputs": [], - "source": [ - "!docker images" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9919b0eb-70f6-43cb-8d64-5bc9e3b2ace9", - "metadata": {}, - "outputs": [], - "source": [ - "# make sure you have the correct CUDAtoolkit version to build latest rapids container\n", - "!nvidia-smi" + "Now that we have a fundamental understanding of our workflow process, we can test the code. First make sure you have the correct CUDAtoolkit version with```nvidia smi``` command.\n", + "\n", + "\n", + "Then starting with latest rapids docker image, we only need to install `optuna` as the container comes with most necessary packages." ] }, { - "cell_type": "code", - "execution_count": null, - "id": "cd07ad98-9d9d-4f9f-ba4e-1b0a1646cc4e", + "cell_type": "markdown", + "id": "0edd2279-7d14-4300-8617-da6abd0fea81", "metadata": {}, - "outputs": [], "source": [ - "cat Dockerfile" + "```python\n", + "FROM rapidsai/rapidsai:23.06-cuda11.8-runtime-ubuntu22.04-py3.10\n", + "\n", + "RUN mamba install -y -n rapids optuna\n", + "\n", + "```" ] }, { @@ -247,23 +232,14 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "bff0ccfd-dc68-4c1a-8a31-e74a5d02b8a9", - "metadata": {}, - "outputs": [], - "source": [ - "!docker build -t rapids-tco-benchmark:v23.06 ." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ab1959fa-f70d-4940-8204-1946acd0e8bb", + "cell_type": "markdown", + "id": "b8fca11b-bca6-4e0a-bb91-1694ecaf12e1", "metadata": {}, - "outputs": [], "source": [ - "!docker images" + "```python\n", + "!docker build -t rapids-tco-benchmark:v23.06 .\n", + "\n", + "```" ] }, { @@ -271,21 +247,17 @@ "id": "baca52e2-09e7-42f3-bc98-5ee38f9e274f", "metadata": {}, "source": [ - "Executing benchmark tests can be an arduous and time-consuming procedure that may extend over multiple days.\n", - "\n", - "By using a tool like [tmux](https://www.redhat.com/sysadmin/introduction-tmux-linux), you can maintain active terminal sessions, ensuring that your tasks continue running even if the SSH connection is interrupted. This allows you to resume your work seamlessly, without losing any progress or requiring you to restart the entire process." + "Executing benchmark tests can be an arduous and time-consuming procedure that may extend over multiple days. By using a tool like [tmux](https://www.redhat.com/sysadmin/introduction-tmux-linux), you can maintain active terminal sessions, ensuring that your tasks continue running even if the SSH connection is interrupted. \n" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "7a2b8b7f-1634-42b8-81e8-2605195d2a1d", + "cell_type": "markdown", + "id": "77a87780-7a19-4c15-a942-477d8578e8ac", "metadata": {}, - "outputs": [], "source": [ - "# start a tmux session using this command\n", - "\n", - "!tmux" + "```python\n", + "!tmux\n", + "```" ] }, { @@ -293,23 +265,21 @@ "id": "77df8ce3-39b8-41d9-a538-ae404be25b45", "metadata": {}, "source": [ - "When starting the container, be sure to expose the `--gpus all` flag to make all available GPUs on the host machine accessible to the containers within the Docker environment. \n", - "\n", - "Use `-v` (or `--volume`) option to mount working dir from the host machine into the docker container. This enables data or directories on the host machine to be accessible within the container, and any changes made to the mounted files or directories will be reflected in both the host and container environments.\n", + "When starting the container, be sure to expose the `--gpus all` flag to make all available GPUs on the host machine accessible within the Docker environment. Use `-v` (or`--volume`) option to mount working dir from the host machine into the docker container. This enables data or directories on the host machine to be accessible within the container, and any changes made to the mounted files or directories will be reflected in both the host and container environments.\n", "\n", "Optional to expose jupyter via ports 8786-8888." ] }, { - "cell_type": "code", - "execution_count": null, - "id": "00b59cf6-963e-4a08-85f7-267b83a99cb1", + "cell_type": "markdown", + "id": "49617e76-c35a-4caa-a07a-6bfaefd22c4b", "metadata": {}, - "outputs": [], "source": [ + "```python\n", "!docker run -it --gpus all -p 8888:8888 -p 8787:8787 -p 8786:8786 -v \\\n", " /home/ec2-user/tco_hpo_gpu_cpu_perf_benchmark:/rapids/notebooks/host \\\n", - " rapids-tco-benchmark:v23.06 \n" + " rapids-tco-benchmark:v23.06 \n", + "```" ] }, { From 15cac8c1f1f33f99d88fdc179b648b2633df3fec Mon Sep 17 00:00:00 2001 From: skirui-source Date: Wed, 2 Aug 2023 02:47:57 -0700 Subject: [PATCH 09/15] update index.md --- source/examples/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/examples/index.md b/source/examples/index.md index f92b927b..b89d250f 100644 --- a/source/examples/index.md +++ b/source/examples/index.md @@ -13,4 +13,6 @@ rapids-sagemaker-hpo/notebook rapids-ec2-mnmg/notebook rapids-autoscaling-multi-tenant-kubernetes/notebook xgboost-randomforest-gpu-hpo-dask/notebook +rapids-azureml-hpo/notebook +xgboost-rf-gpu-cpu-benchmark/notebook ``` From 10eed073fa450d6d592fb6159bbcb5e3595c6a40 Mon Sep 17 00:00:00 2001 From: skirui-source Date: Wed, 2 Aug 2023 08:21:24 -0700 Subject: [PATCH 10/15] convert cells to markdown --- .../notebook.ipynb | 39 +++++++------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb index 6d48c168..65a1b788 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb @@ -76,29 +76,18 @@ "* Actual departure and arrival times ( DepTime and ArrTime )\n", "* Difference between scheduled & actual times ( ArrDelay and DepDelay )\n", "* Binary encoded version of late, aka our target variable ( ArrDelay15 )\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "727b5a5b-87cf-4ff6-84f2-4485c7c0470e", - "metadata": {}, - "outputs": [], - "source": [ - "# configure aws credentials for access to S3 storage\n", - "!aws configure" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3bd10f79-23b2-440e-89b4-f6e4a5c9f3bb", - "metadata": {}, - "outputs": [], - "source": [ - "# download dataset from S3 bucket to your current working dir\n", - "!aws s3 cp --recursive s3://sagemaker-rapids-hpo-us-west-2/3_year/ ./data/" + "\n", + "Configure aws credentials for access to S3 storage\n", + "\n", + "```python\n", + "!aws configure\n", + "```\n", + "\n", + "Download dataset from S3 bucket to your current working dir\n", + "\n", + "```python\n", + "!aws s3 cp --recursive s3://sagemaker-rapids-hpo-us-west-2/3_year/ ./data/\n", + "```" ] }, { @@ -267,7 +256,7 @@ "source": [ "When starting the container, be sure to expose the `--gpus all` flag to make all available GPUs on the host machine accessible within the Docker environment. Use `-v` (or`--volume`) option to mount working dir from the host machine into the docker container. This enables data or directories on the host machine to be accessible within the container, and any changes made to the mounted files or directories will be reflected in both the host and container environments.\n", "\n", - "Optional to expose jupyter via ports 8786-8888." + "Optional to expose jupyter via ports `8786-8888`." ] }, { @@ -295,7 +284,7 @@ "python ./hpo.py --model-type \"XGBoost\" --mode \"gpu\" > xgboost_gpu.txt 2>&1\n", "```\n", "\n", - "The code above will run XGBoost HPO jobs on the gpu and output the benchmark results to a text file. You can run the same for RandomForest by simple chaning `--model type` arg and chnage mode to `cpu` accordingly." + "The code above will run XGBoost HPO jobs on the gpu and output the benchmark results to a text file. You can run the same for RandomForest by changing `--model type` and `--mode` args to `RandomForest` and `cpu` respectively." ] } ], From df105af6e8d89252b3383da146a4198a3513eb73 Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Wed, 2 Aug 2023 13:12:35 -0700 Subject: [PATCH 11/15] Make the benchmark runnable on CPU-only instance; speed up CPU RF (#256) * Make the benchmark runnable on CPU-only instance; speed up CPU RF * Fix formatting --- .../xgboost-rf-gpu-cpu-benchmark/hpo.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/hpo.py b/source/examples/xgboost-rf-gpu-cpu-benchmark/hpo.py index 139b7b6b..7898974a 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/hpo.py +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/hpo.py @@ -4,12 +4,8 @@ import time import dask -import dask_cudf import optuna import xgboost as xgb -from cuml.dask.common.utils import persist_across_workers -from cuml.dask.ensemble import RandomForestClassifier as RF_gpu -from cuml.metrics import accuracy_score as accuracy_score_gpu from dask.distributed import Client, LocalCluster, wait from dask_cuda import LocalCUDACluster from dask_ml.model_selection import train_test_split @@ -40,6 +36,8 @@ def ingest_data(mode): if mode == "gpu": + import dask_cudf + dataset = dask_cudf.read_parquet( glob.glob("./data/*.parquet"), columns=feature_columns, @@ -61,6 +59,8 @@ def preprocess_data(dataset, *, client, i_fold, mode): X_test, y_test = X_test.astype("float32"), y_test.astype("int32") if mode == "gpu": + from cuml.dask.common.utils import persist_across_workers + X_train, y_train, X_test, y_test = persist_across_workers( client, [X_train, y_train, X_test, y_test], workers=client.has_what().keys() ) @@ -94,6 +94,8 @@ def train_xgboost(trial, *, dataset, client, mode): ) if mode == "gpu": + from cuml.metrics import accuracy_score as accuracy_score_gpu + params["tree_method"] = "gpu_hist" dtrain = xgb.dask.DaskDeviceQuantileDMatrix(client, X_train, y_train) dtest = xgb.dask.DaskDeviceQuantileDMatrix(client, X_test) @@ -125,7 +127,6 @@ def train_randomforest(trial, *, dataset, client, mode): "n_estimators": trial.suggest_int("n_estimators", 100, 500, step=10), "criterion": trial.suggest_categorical("criterion", ["gini", "entropy"]), "min_samples_split": trial.suggest_int("min_samples_split", 2, 1000, log=True), - "n_bins": 256, } cv_fold_scores = [] @@ -135,14 +136,21 @@ def train_randomforest(trial, *, dataset, client, mode): ) if mode == "gpu": + from cuml.dask.ensemble import RandomForestClassifier as RF_gpu + from cuml.metrics import accuracy_score as accuracy_score_gpu + + params["n_bins"] = 256 trained_model = RF_gpu(client=client, **params) accuracy_score_func = accuracy_score_gpu else: + params["n_jobs"] = -1 trained_model = RF_cpu(**params) accuracy_score_func = accuracy_score_cpu trained_model.fit(X_train, y_train) - pred = trained_model.predict(X_test).compute() + pred = trained_model.predict(X_test) + if mode == "gpu": + pred = pred.compute() y_test = y_test.compute() score = accuracy_score_func(y_test, pred) cv_fold_scores.append(score) From 2296c6a541fef34732311b183337118f651179b3 Mon Sep 17 00:00:00 2001 From: Hyunsu Philip Cho Date: Wed, 2 Aug 2023 13:17:41 -0700 Subject: [PATCH 12/15] Fix rapids_related_examples extension --- extensions/rapids_related_examples.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extensions/rapids_related_examples.py b/extensions/rapids_related_examples.py index d14a9e60..ef52bf3e 100644 --- a/extensions/rapids_related_examples.py +++ b/extensions/rapids_related_examples.py @@ -72,6 +72,8 @@ def get_title_for_notebook(path: str) -> str: if cell["cell_type"] == "markdown": cell_source = MarkdownIt().parse(cell["source"]) for i, token in enumerate(cell_source): + if i == len(cell_source) - 1: # no next_token + continue next_token = cell_source[i + 1] if ( token.type == "heading_open" From a16ed50e8030e2d32159010e33fb867da1b6afe5 Mon Sep 17 00:00:00 2001 From: Hyunsu Philip Cho Date: Wed, 2 Aug 2023 13:26:58 -0700 Subject: [PATCH 13/15] Fix rapids_notebook_files extension --- extensions/rapids_notebook_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/rapids_notebook_files.py b/extensions/rapids_notebook_files.py index eef81d94..66d68ef8 100644 --- a/extensions/rapids_notebook_files.py +++ b/extensions/rapids_notebook_files.py @@ -12,7 +12,7 @@ def template_func(app, match): def walk_files(app, dir, outdir): - outdir.mkdir(parents=True, exist_ok=False) + outdir.mkdir(parents=True, exist_ok=True) related_notebook_files = {} for page in dir.glob("*"): if page.is_dir(): From 62f80d419d92db1348367c326e60379d7c0a60b5 Mon Sep 17 00:00:00 2001 From: Hyunsu Philip Cho Date: Wed, 2 Aug 2023 13:33:33 -0700 Subject: [PATCH 14/15] Fix all Sphinx build warnings --- .../notebook.ipynb | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb index 65a1b788..9d524218 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb @@ -5,7 +5,7 @@ "id": "a51c95d1-b447-4f1b-9571-cf597ca93ef4", "metadata": {}, "source": [ - " **HPO Benchmarking with RAPIDS and Dask** \n", + "# HPO Benchmarking with RAPIDS and Dask\n", "\n", "Hyper-Parameter Optimization (HPO) helps to find the best version of a model by exploring the space of possible configurations. While generally desirable, this search is computationally expensive and time-consuming.\n", "\n", @@ -32,7 +32,7 @@ "For the purposes of this notebook, we will be utilizing the [Amazon Machine Image (AMI)](https://aws.amazon.com/releasenotes/aws-deep-learning-ami-gpu-tensorflow-2-12-amazon-linux-2/) as the starting point.\n", "\n", "\n", - "````{docref} /cloud/aws/\n", + "````{docref} /cloud/aws/ec2\n", "Please follow instructions in [AWS Elastic Cloud Compute)](../../cloud/aws/ec2) to launch an EC2 instance with GPUs, the NVIDIA Driver and the NVIDIA Container Runtime.\n", "\n", "```{note}\n", @@ -79,14 +79,14 @@ "\n", "Configure aws credentials for access to S3 storage\n", "\n", - "```python\n", - "!aws configure\n", + "```\n", + "aws configure\n", "```\n", "\n", "Download dataset from S3 bucket to your current working dir\n", "\n", - "```python\n", - "!aws s3 cp --recursive s3://sagemaker-rapids-hpo-us-west-2/3_year/ ./data/\n", + "```\n", + "aws s3 cp --recursive s3://sagemaker-rapids-hpo-us-west-2/3_year/ ./data/\n", "```" ] }, @@ -204,7 +204,7 @@ "id": "0edd2279-7d14-4300-8617-da6abd0fea81", "metadata": {}, "source": [ - "```python\n", + "```dockerfile\n", "FROM rapidsai/rapidsai:23.06-cuda11.8-runtime-ubuntu22.04-py3.10\n", "\n", "RUN mamba install -y -n rapids optuna\n", @@ -225,8 +225,8 @@ "id": "b8fca11b-bca6-4e0a-bb91-1694ecaf12e1", "metadata": {}, "source": [ - "```python\n", - "!docker build -t rapids-tco-benchmark:v23.06 .\n", + "```\n", + "docker build -t rapids-tco-benchmark:v23.06 .\n", "\n", "```" ] @@ -244,8 +244,8 @@ "id": "77a87780-7a19-4c15-a942-477d8578e8ac", "metadata": {}, "source": [ - "```python\n", - "!tmux\n", + "```\n", + "tmux\n", "```" ] }, @@ -264,10 +264,10 @@ "id": "49617e76-c35a-4caa-a07a-6bfaefd22c4b", "metadata": {}, "source": [ - "```python\n", - "!docker run -it --gpus all -p 8888:8888 -p 8787:8787 -p 8786:8786 -v \\\n", - " /home/ec2-user/tco_hpo_gpu_cpu_perf_benchmark:/rapids/notebooks/host \\\n", - " rapids-tco-benchmark:v23.06 \n", + "```\n", + "docker run -it --gpus all -p 8888:8888 -p 8787:8787 -p 8786:8786 -v \\\n", + " /home/ec2-user/tco_hpo_gpu_cpu_perf_benchmark:/rapids/notebooks/host \\\n", + " rapids-tco-benchmark:v23.06 \n", "```" ] }, @@ -280,7 +280,7 @@ "\n", "Navigate to the host directory inside the container and run the python training script with the following command : \n", "\n", - "```python\n", + "```\n", "python ./hpo.py --model-type \"XGBoost\" --mode \"gpu\" > xgboost_gpu.txt 2>&1\n", "```\n", "\n", From 577e35e68e04477d5f4f87efc2496c17dfd863b9 Mon Sep 17 00:00:00 2001 From: skirui-source Date: Wed, 2 Aug 2023 23:59:45 -0700 Subject: [PATCH 15/15] add results, will be updated once new tests are done --- source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb index 9d524218..0823ac3d 100644 --- a/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb +++ b/source/examples/xgboost-rf-gpu-cpu-benchmark/notebook.ipynb @@ -11,7 +11,7 @@ "\n", "In the notebook demo below, we compare benchmarking results to show how GPU can accelerate HPO tuning jobs relative to CPU.\n", "\n", - "For instance, we find a x speedup in wall clock time and a x reduction in cost when comparing between GPU and CPU EC2 instances on 100 XGBoost HPO trials on 3 years of the Airline Dataset.\n" + "For instance, we find a 26x speedup in wall clock time (0.71 hrs vs. 18.9 hrs) when comparing between GPU and CPU EC2 instances on 100 XGBoost HPO trials using no parallel workers on 3 years of the Airline Dataset.\n" ] }, {