From 7ea82981ea45718f314613b68c84fccedf0651e0 Mon Sep 17 00:00:00 2001 From: Jeannie Finks <74554921+jeanniefinks@users.noreply.github.com> Date: Wed, 12 May 2021 08:51:08 -0400 Subject: [PATCH] Jfinks scheduler (#101) * update Flask examples (#96) * update Flask examples * adding logging, adding post/preprocessing client options * update docstrings * update readme outputs * default to multi_stream scheduler for serving * Create scheduler.md new content for single and multi-stream scheduling * Add files via upload Files that go with new scheduler doc in review * Update index.rst included scheduler doc into nav tree fixed minor formatting issues w/ lists and markdown * Update example-log.md (#99) optimized link so it would convert from .md to .html properly as it's resulting in a 404 in its companion html file * Update docs/source/scheduler.md Co-authored-by: Michael Goin * Update docs/source/scheduler.md Co-authored-by: Michael Goin Co-authored-by: Benjamin Fineran Co-authored-by: Michael Goin Co-authored-by: Mark Kurtz --- docs/debugging-optimizing/example-log.md | 2 +- docs/index.rst | 2 + docs/source/multi-stream.png | Bin 0 -> 31497 bytes docs/source/scheduler.md | 55 ++++++++ docs/source/single-stream.png | Bin 0 -> 17490 bytes examples/flask/README.md | 14 +- examples/flask/client.py | 98 ++++++++++---- examples/flask/server.py | 165 +++++++++++++++-------- 8 files changed, 247 insertions(+), 89 deletions(-) create mode 100644 docs/source/multi-stream.png create mode 100644 docs/source/scheduler.md create mode 100644 docs/source/single-stream.png diff --git a/docs/debugging-optimizing/example-log.md b/docs/debugging-optimizing/example-log.md index 707daaa52f..4ea303865f 100644 --- a/docs/debugging-optimizing/example-log.md +++ b/docs/debugging-optimizing/example-log.md @@ -16,7 +16,7 @@ limitations under the License. # Example Log, Verbose Level = diagnose -The following is an example log with `NM_LOGGING_LEVEL=diagnose` running a super_resolution network, where we only support running 70% of it. Different portions of the log are explained in [Parsing an Example Log](./diagnostics-debugging.md#parsing-an-example-log). +The following is an example log with `NM_LOGGING_LEVEL=diagnose` running a super_resolution network, where we only support running 70% of it. Different portions of the log are explained in [Parsing an Example Log](diagnostics-debugging.md#parsing-an-example-log). ```bash onnx_filename : test-models/cv-resolution/super_resolution/none-bsd300-onnx-repo/model.onnx diff --git a/docs/index.rst b/docs/index.rst index ade7ccf00d..b5eac02105 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -66,6 +66,7 @@ For example, pruning plus quantization can give noticeable improvements in perfo The Deep Sparse product suite builds on top of sparsification enabling you to easily apply the techniques to your datasets and models using recipe-driven approaches. Recipes encode the directions for how to sparsify a model into a simple, easily editable format. + - Download a sparsification recipe and sparsified model from the `SparseZoo `_. - Alternatively, create a recipe for your model using `Sparsify `_. - Apply your recipe with only a few lines of code using `SparseML `_. @@ -121,6 +122,7 @@ Additionally, more information can be found via :caption: Performance debugging-optimizing/index + source/scheduler .. toctree:: :maxdepth: 2 diff --git a/docs/source/multi-stream.png b/docs/source/multi-stream.png new file mode 100644 index 0000000000000000000000000000000000000000..769133fb0d5d903506488ec6fbb584813cf27bc2 GIT binary patch literal 31497 zcmZ^~19&FS(l;F2wry?9jcwbuZQIGlwz1jRwz0A8Y@F}j|2gM*&-+~8d-pXnb@z02 zcU5=KR9F2bQbA4}9tH;n2nYyXQbI%t2naXbXA7>GtS9J8#nia;m`BM41;oRJYIZUk0R^k%sA zW+bhL)d#I+OI>YD+u(evAd3}&S!-05oO2lqQG&K?!fn3e-_8O78z^Wc2T(w!n(E1O zOagg&J2^8B22gfzy`&L8P^8HA@}H*#_V*Fk)9l}Embc&HcBhZnkn3*;l?w~c1}6pw zXxsZef>(>?l?FRvU@7vLr!#cymy45-_q2`lQbOo%aU0}Q7Fl{5Ubbwwq`5h48z(i4 zvMtZ)n;n0tpx5xNA?k8@+NCWmV2hwJPZ9wWV5=yKKy$yRm@2W6o$^;7tGO=YRZv-5 zH4vTpMBt6zGHc@9-}VVQ;2HRRJvEf5A_R`P_v%?&X5#M{p`Q3#SZ;ylr+0GqsDPU%|4o7)sY4`<611j?ca|w;fyT4LoJ#v@K|HW zT?H<3BlUnhJq6Bzbp3jW7vGePrz7NE*5LO8wqAB$1eZ?81dLLnsfMJPtSk@>fQABs z1V#Y@1yI0%1O$u&1pW^V1Y`t^`#-c5Fx9`yfB?!c0|CxJ@PNY!nB^ZD4L}EkTLOXo zn>Ph;{ri&u9DrK?brqMCRUl$vU}j=q<^%#_VPawCW@6)J;Ur@I#?8jW&B6`@1dTdjJFd$j181XQ`s$q9H58ZESDLU}$1* zWXj-S>+qKgh}VM~K-!wR7!rBd+SoaBd+?F`Q-T{n|IKD3A^N9?i!~pK2B3?=_D-fm z-x$~!m`M0xh=_=IolMNQl|;n;%?|j-M`GdP;=s+w=$;Lgfm?_|!%%*DmU$i%|P z!a@%yLGSEo=VItVZ|6+19C(cgLvjqF`r_((|pHuOKgfA!PF z((M1VWas>Eu>b-x{=LJ<%)rF>|Ho$PVfp`H`+MjA$p+BT|A_T}HRfUH@Gqu+l<}`V z^8W1sx4e_3DL~4@u78WG+F82rv;3Roe`Ws{*WV6v z3)|b+J1ILD8k_!Ylm!u*p9m@)h?0M7r+K7N?EAGPy9 zK!QM$B0?%2z!$oZ1}dVc1QLk!4>Y2JUe}_6mP%+yDuQT8B12)l@`}i4Drkx}`BV`k zMDwCZU=<{?38fEPJjBI`f}bx(ri_f~2UognWbTHT4A))sO&v|wQy0^d(`J$Tg4`Bx z|4D5Tfz=F0D{J2%*#4csKz{_DU zdhv4OC$!fSon|BFMCHOI6Hw>N^YiM^nRqO4EU>!)P_txs^N`?jb=(S;W;43lIzm8L zEpt8~C1^75xPUvo+?jH_oG`t-K95V0HIek^eDqsgF4t(M@QJh2=y&sS?eASxYY)t0 zwP`asoGnswdmo|I-L<$3<`^TqqN6%T!=TZLkV~gIL9J7RYBrisKVH1B@h#M0=J2?0 zA&rqWxq@X$UcDNc3>s_Z&}%d4w!0*%eRQWy=LyJz!=O2ELUAhsgza^Oi}Ce`^X0m= zKAUgh*o1;Dseb>!&RSnwW>+9{y{38Rp+2|L%}o%1t)h;g?EXouUi-V*;l;~pb4Tw7 zt35F|7+C45U>O|Q!@BF)o0U5byAM&Q-oDVCVWx)F2-%RCNgPtMY~AXr#&{-ch!%fm>K6X0 z3>kGC&P7Tf7&wH|sgh|q6Qg-ha8O3ziI5a>0WT1et|55;d$!eLb?J~gHHkE(1>A3J z2p9~S@1jv?1^uDH)GAKfpQcTJ@Z88k4hM=qcQBc z%4R9Ws}<;Xg52IuOnbnWjow&K#^XDil)-^b1)^|TtBsDL_<7|gA*A#pnlxx)nRj=8 zO0u3}{hcQqh5&Ntta)vnP()J@M?CkArv3tKKgUWZjW z9&PTIXU%EmOWgQ!{p{M<=&%ohU>S zyX~UFf7VkcCSW;vT(6#{S^u-%+x*MKzTy7Y^aDpma3O~{$sR6Mcbebr$MSH4# z(kmfX8soLBmVhRT2_bG|_w!)Rww{+!_jW0k-#@ah71iFe${-dRe%lhbJ{RHwShcanl&xt^>`#mpzh-!d>@Q394{y z&s;F8V6TRF4{4#6QFV2M{u>ZAO3l#q$o8}J!YBn6u|m=rtrG2ooDs9QDha7Qp6^I# zo1l+ihY~8ezA%NncOV2#t3TmS`8macVR21k*>P+^FInc~Qp~+zI0M3+B*gJ@75TsQ zVIOj69iTNnj-!%N(}IJ8M+FOcc)~%!)iX!$><}3uDJm)|c_Z;ysg(WMje6L)2!}mY zipvBpl7JF`=@pNzgjF|I2ppsXfkc4c9XU69ki{Mqv1( zWzfd@^mNnhaT>ESG;m0W2yjsc!6MSC3cH>><)+Y(IJyBv|xe^FE+}=dTt&)Q{^_4vX;>0fh>X z)RV2CRLk+=Yw(I+-<{@LYk0~nbCy+FCBD%l)`8UMHwj@y?X3$;Wl$a_e7-!_ot!ev z%n!qBQ;APsJZzQ6TbwdZK+6C}fU}&dSc!!r;O({tmykE@Poz`pq!L)oC(7mV4uG06 zRoovlC|uXhAkW{>sy9>EZg$2Z28_#LGU}CzZ@88D@#zua?bA$JL!p=&kRIQqN*Y2A+~mGX6J?MAw?v36J{3+%dq zD`rucR%lrYyqbQ}Ma4!UdwIOM;n8Qa=2)db=X3wAJI=KdE|bSk_M4Cz1ot5~U^wLY z$%@o0PoHJ>#Tk!4$Svh9Z&OC@KUuPK(bunoaKR&TNpK-!AMcx7JDgqWDQz~pAI~fM z_QfOkB78Tu7btO?|7vaIbmwup zC;BvwP5#?`EeZW$-1)1BPyFZn80}lRZTRw}!4Ih0@X@nq`R*fNEm=&`__4V zkn@fD>a+A_BA)k01r2$f!ZT0TqBR>W`7PS%V|5k{B=0CgPy_j3fLZ zQFSbL-?7>8kLs(*g~Jb4XBO03y6>Rv%{4+r!+yfv^22japou9Q`hM$wt_51r``z{9 zi$LGp7W8=-7xq-WPlMedW|Kl$J-=tBB3x}tZ}PC`&rmQXH11P=%4~8k78QjfC_Y_? ztkmihY6{2{?~5Z!Z4&yiN5B5D6USuGIS9`6vG_Ax^RSx{*z7eyz212Y=gVUZUZc~U z(C^t8eFu9aV_f>oR`c;jcPW4>J`}Yr9sDXFguI*BDAn$YJCAlQv$9sMInvE4L6bwy zjwICg$kU&|p;uHm1f!galUku2KN2F%kZ}+Dc(R#_7-UIxm4VA%A+K7gS~!LKRa#$; z`+zs|+4HgsTd7+|!5Tv3RVZ!_t_yLx)mY-8dBixuFH(*PIx%{419kT2Vd7z--%_Pb zLWm(fG}mLEs^@z9!%0m%^X&@V?87{Bct76u)qBntXB)x!-97UF?|WFI}_lW_TJ!tR-F-7K0`K z0()fms+ilIaR=KqO^aW8`!I!hy}&wySQ1WnNmW=Dr%O;&_Sz|y7wbQ9BJ3mn`b0SD z!Wx3ZX2&4G?f--pT}HG7m3VBc{b}NbE{@o0C6wp)Eei&HY$kklGoP?bE?>U;hjeH( zMv_n0E4)!@Qz+#zk7p>U;`@lNk!$*tM~= z+|PNvg7r9&v5~2~x;#&XIfJGd}O!#{NCx@ZkCMW zPA9So$D#L_qB;<_8Af4A*!ES)0dBt!ykODkB=TgRb*4oHoDQH;X;jB0C3C9(juQ&R z4g6F#SF%_PmJs~eN@4Eih9aDQ-goC5|1=Pc_PdNMj>B$^TcyrIB8oxeLeR*_M{^2C z)%o1>H8})jn6aGrRvVq6M2hGb#bOw`{F%G}NAyvf5IQIQ+e|8ND7|V2J|C z)@q3o8+V5zBFp9Gw?U_Dw;lSKt;px@ngSCql`B@gRUspqXadRSlTxacEBlh1d5$Ou zX5r|d6vbq=^xCy3Q+l1LxgKo@@O~Ave4Fk>d~5fjH0rl{?T-80tcl7G>-V}R$Jq3z zym^@ZpT8}s5!>M)!k5x{f+yfYEL)vF(FHPCP&p36;HKk|-5$qPaXxmT&fdN=V#_e0 zxxlq2)2i1>evx>P13v+kD5#*tgXJ>$?r?$)g06XZ+DJqmZ%KUZ*Xer_(JVa9p7Yjs zH`09nFkG>}p(DX1?V{!T-9Uf&ZqC6sf9Hsi^&z%Bl}$z)Yf&pds4#g4j4QAR4*`Lt z7R1;C-1E^I)BSElQddi}(Di4lYDWBL5ND3KaW4X>`AlxfFX!WV+PY2>YCjpBaCrY@ zS4iB9avZq9b@j0!S?p9txPZM5Ghg-v95V_y|KUynYW1t^?~Lys@Z=nKq9VUbc|C05}K48j{S4L6ZV>h%iUpD)(Y?=_*0#Lo{xUn`uyNiT%qr)&I33P${bEzyi-SKfqh z)tvMfKA}GoT`QN!9b{FgVuSmZHEYT?cva`UhD>viR;Z-IsM}`5hEa1IfDVMDQz0da z!)($FC-=$w;`uE>Sq+B6E=0NSJ9xQi7Nq6!!>>ZD@nllO%dOkG6iAW`bOl;D{l_;P zy~y8tcGZ9rXsr}p%x=gy1WVX<2PmXvrOGR^pheq&Nm|`3~FL@3kS{z zS>LS1wtep5(GSp}*9-pIYyr-2I^8~*-Cv*HT&)eoh)WLpc zEOU}NMxGmd&f0gK|anT?KT zy^J>9tR)Q9i!NP(d@Oe7e^$$YPdR8j+i}Bo8MMeZV*k3_8jgDH(L@)dBcK&E=Kn#0 z|DL{ok2xx2I-yz$Z=HT$oLMf|chN6c_{^pmv8ZcG5XqeWo;*RDaL`8i^1tl@9fz0yC5z6wAbhMeQovut5T~;aw}C)8?lD_ zJ%t6duhD9Q{dh7bJPR}od_JBrx4DD>|92-MXv{9*pPu?YZc87z1 z5Btm2>-h2b^xhsbGH#dR8dun2qnR%D<3XIe-kn4C;|x@*(M({0)o6Vwc8lN>PuuZj zEgF?UL``gA*qa7OjFg8P?)egmQ z(-db6d2_iF(kj^9eBP#bBWY@z$+xEw#?zS=T+5gF)OVJH(o{MpnyK+iYjxqw=(N(B zubQGYfoaq8kfMj(5ayW0qQdQf64fzp>DhJqiIX>K*0m?DIJ4=mIJq27dMRfOt5j3+ zXMwk`LVe5+u^z^cV5^lHp#d>Cv`uI=D#&TOqF8@BvxVKBkb+6A8~i1E74--{Gm^y4 z;t(bOvC3aYIjYH-l1)dk)KuCDHgqn-ylBG%wfgeSM%Y|VWcUJpIvLPrYXA7N$cB0k z#*$ATSM;xs$c z^EoBbiL-#X$6KKC%M0JjdXntb_SOc&d%Fm>crimC(-u9O)Y~(Aj<5RPP}eqBR7>GBKR=Qofap^^K%dxM<56fIIEly zuf&D@F-LAbE6A7x1Pszf!g>}LN#UXxlYYnw>@+M|X28^sgbuTa8g4lP-nm+AJ|S+G zzlUOK>U$OZalToJOled|89zkf88k*JBRM^Wt3DEoIqnBSYX z8%HE^rjZocLHi0qTwLyTg-zo1`aF+6P+WMCq23a0<+Wz$oegY%%xSQH29Dy<5}c@I zxH*{3^4LJY!LX~zm9ww}nSRSgetc_c%N|2L*NRDnd-4N=cxBNw&V*iIpB?BWidfq} zKc>wfV*h4@4z2~;9(rjm69tBWA&L}R;RnNz^b+jSOsX!yzTRR?DeH4BsnzJix-!z( zh88r09wAJS4ApFeq7f?S18x+X5202fOo-x}ijCrq^POQZ%lwIHL{zXW013=yii55` z;xc!@(4X8iv2-U0m?!{S2sz$-s29yya8v!4U9Xs+X7R4(AvkFpGKO7nJe#T>4A%#! z23^s&0=X8)$FrsW-%)7P-x<4~3za3fgNJ)J!veRuyeEpU4%78ZOJ%bfL(S~go6He? z-)~1KRBlF0KYL}U$F`jGd`2=%#6&kSyn@Yr(g7yX0go9J^4L7&L9iS!MYEX~5I*Qv zJjbo!>tJCGU5Ydm&$xshog$}T44SX%DOtX{DI@OfWrWF(P&b2PZV-yz6=Ba2?i1AG zU@F=RZi@c=SP0vRam3**p5Bla%!O{KBHyUo%nh1=&4&>ZP-du_CGI7`=Vvl~Ie09r zu*7tA6gZusA_5mI;E2%Fv*0}w^v?W8K;ZKiJFZ<;M%rAtj*y2$#er53Pf5Og7&ZW`do zD&N@0bQ^ZD>X;fM*}fmDnv%0Fpn~UBAg0Ua%Nl?MSWY9-D)9zO5|=kd4Dgl5aP|C> zljP{b1hRWKP7G{o5JRQXE(^p{=C!l6^od^*yhRzJN~_x$6>MuVV6g92Kb&+t@=lGj z{@xi1GX;Lg|H6W$)A{_%=@6yFYpzg~T0WO|u~0Oc49O(~FuW0gu$T-AktpOApYJb= z70Q&Ffq-+kp6{b-uE|n#wkr&zl*yK>%x61&rWBL*r;BEk|A2-5AXX z(eqdf(xhOj%#l^Pg?`@4nok{3=Kka#n$6UzA~TRUQ9w-AxZAyix?%zrEMUx-l@LPG z@qJO~K1Rua5Ph1S1FtnAa3D^l^?UpzYpxSYe24$Y=+T)N*uv+B{rD_1xfd+ZjOkT z41mL7kCR;+5e)Muq|63)@00)xbi6>%#VU3A3`waM*Un)5>sx;tMHq@XC-Y}23f-EFK!3M*H*6Zv;b6vSgL;;7A?$AD^H5gJ`))q6@+va_< z#@KY8;k%eVjMB&_AO?i9Y7S2p;fDPI1N@} zdQdg)O=dPQBVd}t=l9{fo@l2@R#9K~J>B!nb;v1;#w_%-7qRH{tm^wd88gTXEu%r2 zAJf!X=}Ud6dV5xRS7j}AOS9?Il7#7rRNc%&?g4$90d)V_d4iNVO7QzIr*u6 z8EwLa56F)CeoxuzO|{F_yXVo}t)EH?RkU_)sCL3^#Pb#r$*+Li%1P?ONXD{8a!3lr z>>x`J`qNU6Q~|CGn(1z?qrEl`NNsF62?^FGPlF-1NcnHV3Os95ra7C&VowLj@=85^ zAA``OcUc15=7+2__;y+jVZ0a1x|xz@D&DNZT^@```J}yPm{Df(BVAV8fOT6gpT}=B z9ml6w8)igaHEaDrP&(57rv!IWkO&|7BlsiS266S>uq@?>D&7~{ov))wPTgSWc&Kd$ ze;hq2>*%93_s5FWFlTJpAL@s>3fkmo-emszuK=5#^z@~*onnQ#O|%*akjbA)B~h)h z0%{KVSl|#SX`1-lE(!OD3Th>qfyv08XojN^?XEcx5C#yih)cNy%XRwlvOm}o2bPnx z)HSU?p!sJ{7spU#vNWn0TqIE`tuvfcCRS+zoz-@j8rgW#UC^>wkiwbl&;nPHxf z$ES}~{5~F68gYbdY3OaOx>?I-cPDfC-lhXi=G{Yp@i2k56?5_2F4skuDpiZFp@5s3%b&p~xs2 z7+KP`?;^)icunYl!q052l5-`>Y(3UnZ+N=gSZKCf7))lIMhn#T@Tsm)T=u4hacSbi z1I<`0|AQ>PuxEpqZ5o%K86J+98$GoEHz*q7`TZ%E&l~0Gc(GbD`RQtt7E;BMVh@_M zXd{16t#`02&rj;Q=c~KTVuaZ0ho?#RAK~cwP8XTo2DB`=4wq9BzTOx_84<|Xz?gm@ ztPm9KsSM`e9irRcznoiRW3`SG6-krb9<3mf+}dQFM>(%h5ilL?*g!HOmLrR~);HAx zQSRH|4In^R>KFj}7vuAK$Pb1@9N$&IUZvA@#W5CSO<)sq&0h>vb54=|+y-W}1ZI_MY5kPZe`$JVmj($*dx>1;n4Cax793V%#HH3`o zaoDfB1L3y&3D_>Yp|F?~8Tzzh4Q`iOj;i;o#_^ImfVcP%Yfa69oXp^ylh) z6mqHApWA-YIoz&EX~u9}w;%T)&E9ILFXXetsD=TivknsIe`W-}BFB2Fb8>9xV+6#G)GvPi>hW zSY2IZySB8#jv$G1KAw(^j*60M$hiQei#t4LKve}7=&CjAM+C{j6PBq+#u@*FVqxq7Ca)p=8p&pp^7zzZ0?Uj(s7 z)c>8SOf#_L-UI>80wRfPm|2l=l8}|BKH8Wv#?gj`#&bR?Qp1$U#>>VS$p7=jys%Z< zQ}AojE79lWZf)n6l*4A~_T;FCvH9-?H>_))nm7ms&)*e+hj7hF5IT zhm5j1faQ!$NT!&p^_yWMcYb*mj` z5)@?d+`x@_9ovm;E@zwiRo}MDE#$tgWVREKdi9|tgDXs^?yjXJfFoqzV7{t9gV~tw z4>>&+)MQ^EWlSmp4yYNN1u}*|w!1WyLViFS$lAsPR|L7VnG{Jz=Y zURIXtB_-GkO+4-H2^jAojggg_QWcYJyF#9_pH)*-!p*2>D3SnFyBec{kvnY08)%Ug?8 z3BUd383gJ<;{F|q74SlL0!%Bix$Ip8_WIu+0lH7Th?5_)Nzu@)^=|xOgVWwkyL#Bj^{kOLqwx9C|Nek#>4PDS@Xdx zdXdBZ8$>h00RzFOzW;|R6l#`9J2B!U*rA&Wn_%wr3tmqb!F55 z9(lM*-5a+zfDw(vyQBBZN%TkMJWJyyQr*Yfbx$jWpie(8$&ksiYT`0!Jn#1ox68&p zr$dNvm%h$6%_g&2H0dOFfI*>BEP-rf+E&~b^3op)``3fLfm|;MV`>2A5Gy?Hpn(mt z1+9L|0KV`Xe_QKy#E~|vD-U^tamtSvgy2pLG07^Gv9MgOIB?`PkbMMW^}7W`(bcHz zo^^)TSX-#sew%;Eh|H~o08%{m{gCUUPZW}NUzJtK7w|^6_zlb0nNZGOMZ$HDReTHs zX13Kk2UrZn+edE2ILPe=r9%M=pd|?S+#=`8wPic`Hm#(b3xOY=uIQ`fn5ekIfN)5m zT8;Ktg;(w}@g&ttO4398OIduwQI6{Qx|lMUZ_d9c+8& zPavXv+y+Q+N#^7p{fZS`-Qk|>TiXDd)C4jf{35sT_)-{jIt9D=`FSwu&!;$_yoO>c1j5Nux3HuV`J!h90^I7QQX~*5W?=_al266 zMhz;CEPR7!kTO_?QC5oZL8=GEFXS+7F>Mjz^zQWk`c!E&k>si#*FgwL=D6ETbVc8} zTx$x4fWs05ETixM`;Aum21B4!dhKPMoRQ6-`l6NB&D4Rq~ROVEw z&ptd+9X0;cdq5McI}!f;2Wuh1UGPzn0kcA$+#WXudz*7KYsR&!q=B;GXBpa5CAe%B zg4Nazzhcp8)zAh4k8{1{#?z>k#e8^w!9~E@tT#($V5u=dSrBULjzvu`WJ2bJNVHfj zkz~*4wOFZI_WXPw`;khm$Lkvm1-qT?XzGX|qIKJ|krd{#;!m9lZqwT&uYP-GB{$;h zbX%!=)Opl)u&2tsL!Ci`pe}BzB*$?>$qD{(XZUWUc0t6c8_0y`csD%G);_g|y%A7> z3yuexsiYe>uk!NH{$sABhe(RI7&-?mHS?JP~y|o|e`3eMATBk7Px(H4-QD z1h(C3e>QdZJHZzUTx6>U%KBBT;cMAaYgqaYiOD`4l4oH%@^}0M4@TNDdNrA)`h0zM zK0rj-E=H&|41mTG(fEEWbkrAOH((rMY{IP@NVWpw;1vkFdKMmYw)J^NHN|hQC6M)x z0F|8gs+$|fea=2O3PyzWQn#h~8Bl1`NQezf$w+4M=b-D6IcETazwsNqS0Z5Rv-NZ_ z4zDeRObALRO5ppT==JsSC&%m0As=2Lt->VemdEb=rgjEoC+k2F!=N2tqwN5$o147y zJ~zhaZ-4fKgt?!i!e7!M0m|MgB4>f|9Bz<{PW4>ZU$q=X?A&x!K?Y1p|0zz|2TPSi zUp03TpMOg&rF)zNR!yD>#9(HCfdiEr$d!HUysk$M;DjQ74s1lsXelB%F#(J5iXVs`p%Gu@tZ*cE)6ep8M!L6g?-^t>FeC(2Erl#aXd@^Uy#O zC@Eqe!?2i;LwQ?i{gvd{Mq;xaanY)^RFBWa{ajAIrYPLK?=OK>cSd`Ja_)xje+^bWhD zf{$3xK6tHJ>3!dTQbVNz6;{`h)SxfM^K7v~soLxD3Cw``Wgt;n13YOK|!??3IGG0VjA9G!uA4y+njN8ZpoatI$pN0&$=Kv02`n} zz-wiE65R{g^lCgDmi$aatXgmq-e6nS2)ShM>R47G3b~xKN*&34>l?$GAxlhb093tF z@K(M`h0@){I3XS=?4QdTt(GJs2l8N9?Njt#lU+^s)A^E9)skAuWB>-u-u{R73xe%{8Vk*Ka#*x)%+7iY;xG0bQi0LoR&}`5S=|2vrpJi!;=nr&Y%WMCfpX z&Ny}2N&Hv=P_QA`SuX}#(aycJD&<5+d1m6FS43fE{b4!B4q;H=&!qx#L2T8gVI3)W zr}3E29jp6;QAM(qrx{K{Gy7t&hQZYax_#dUSzu`q!8nx$X@G#Aogk;dGJ*vhjv}0o z*9ftML?H3X#JdoF6eqJlL(s7x(YH!1d3fI%$x^!(h+O^t)#O50_TCpnNkr63;S~s? z4elOcjAj@Nn><3`Eu%)fDhQQlQybFTE4bH0XR+1ksRXx2dg7S>;P%a6I8L~>R|3hw z0^LWK3&y|;;H^g!i{_(&(ol@WV*Z}NoEsbzg_LMs zL5nma-q%&UXKC&y(I*qFUXxkugh4~yfQ@7*yz_X}xT*os;P@1;HvvYnAzqg?p+O(G zW4hCN#1bqVf`MC1{-7n&SEjnolf%^mwrwZ@l=!hxnQtlF)?1`$)NcK#;8o0&@vw33 zOl1VU@Xa_JMg}{F=Rs37c-ahos4S1$j)jpB%?) zR@N2spQ!RaW}-OzP&h&<)9j+prwx15-_E@-(6>ti`~Jd^u{K5BtL!fDr50h#nV z(dEXP4e_9*?NJFGAc))v_s28QStuEBm~}dV(*uX~o)gJX;hM2b~ z;Q~=yw7eTAzdfAy2BDER`XotoQaz*tD%f0X)rKjgm%-K()<4HM?8S}Ist@KTllx5; zE)B>#M=YvPxv5N1UsJI7N6Mf!EEU_*QL7D`+NHugwNw-!)Ze;cP3+DgERgJ$00< z%tIRJRYA8~avVi6Iaw73nuB}i2R3&IEg0ldsO_N52SDC?t1;9Jpb`!TLNxFLOiTEY ztWNBD;a)mnsX;Kw9F~%B1hfEO3?#5lQp)}yt77H-dRUOZ=oa{CYY&cvpGQrC%dA+C zsY!L4&owP2P<;@1g1}945&r^+w29c8mSmS508bkL0hfLaVd{@kEFY+OTGYv_5hj49 z2gf5?C=)GUeR;fsaid#k3k8eqYFdCTK>L3Im4epWvOPR zPXf(MGkxyvnuR_uyizmdH7|uBRWvu~uWF$`?{Mv-U7{Ogu@f-5ChLdC$M((f8)m&X z@)o#1S*s8iM8j&jt)-!&^;tQ!xr>dWVu+=&>2RRG`|+{GaEK2xxCA)}!36~q}f}jo5-wwar!8+XTnYIr) zBw#w+VUAKV-8K^Jdm&!%NQg$G7c5ZD%pOXoi;^YgoC6~8;=#RC379+zdSEMmpgDtb zaxK@I%rVVF6PEU?p(2a`jyc9@kX>%DqfP}XW8oht8*NR#R)>)1LgGtGuJfDbeL4*m%y|2K-;|_aLxlzpPw?KS(q- z0~$#3^$v#QLz@loVAuyEu@JG4dEajaXx6Q34SbM(3h9GlzCk&(1VzGbbtREYuRENM zh@=tfoZ&`%d+=DP-O9(~5Cm&DmL0SN8%WOV4F^;|RvQqJ6tTb9Qgy@6|5vzW#R2zus=i!}sMxi1V~ebmao{ z!D~#=wdzjf>UCkL#bnOh#o>qMdi}x)ZVqck7B-jhlRw{bdoq7~iAD*4A+>m2)MRr# zB0GB*f%IygN`8wRpRuBuxiSPw_eS`BA> z`OW%gQuoK_F{4V07P+B}6}stp^Co;zfG`G)LOoElJG;V|(vYcI#A-7cL4bc5FeAr0 zlr?_a%kT)sF?y~&^Zm$34A39^S=nA*v@M-@d>)G1>Caxo8sAwLkI9!drvK__2_Hp1&3+# zIT_;5xdc5AujF6+e9p$F(T~L-{ue6*LlSRV_PC>&Tq%3_Q|WK#>;;$F%X1m=-9_Sr z*P{HqD>_W({IPhibX}iM#}8W)!GWLGQ@4w6zXB&a-%nxkSlON~S2A*I)7+-sPPNbO z`S9MAPshR;g499QMEE&*65ALcr(zo-t(qXAIqbz_)!UOu~LAqQey*J zsZfE}a(G`a@a2)j6u&9ns&ER3?eql=^$P+q(lU~hYBmR9B1nIKi$`OPx$emk3V$Lj zv}P`&?&Sy=B7{BN#fYDqgZ0h;JcphG9`8F((I&psz8c~Hh+M%r89G?BN%?KLywS;H z?_6NKBKP6A1Z;3MdIOc|49Fdv=g8!CF$6+c@(J`^^flM0L+k*OsHomm{n)$6uS#AJ zYkSX0T!G+tcdRcj6SOdWE#js@LAA8i^N8N_!g2p zlT$XK9XU{KYQ95B+A|d>EY%6?=bxUhxfpC#6J-JsNnekia9VDx6Uur;q$-+-ru_ZD zX~;ruCV6#`2cldQvzc7_&l;ld!4ueE)x!-=YR@YBHMOI`-sm?3dI^jXFPO7u$_(ZuLz1?corN@7L5wXFHb^lLYXJ00YP|#OcT@~FVJA1=%IHoa&-=p?09rm7v?7gB zQwX2?XYzxqf8Ak;3Bs@KZdb^q3iasWnCAoxx-8M5E5m+V0x^0m0isif)HrmFE8b2W z6F!J0M2aMz*lRkIAOJXa!MljRa7@5QE*udb-MXOj*=Db6p;-OJm!;%#(W}2=+-{E;CwNH> z5I#+!O|!|)BraVV9yg(FjSeX!xH>gt4(DU)O~#Ht^6n_a{w$|3!xEN(st@WH0md+a zQms@$K0*+NsTS;0$Q9^JrZoMjW5qKQZ*my)eZMC|RP=gmXqbh0@#fQv&Wwa&m=;{` zv=QBdJZMRQh&7n|Vafb@miEg#J<`Dq2LRlR#kzw#Jm3>rvSv4+mcslr>kY}&Y+8ld zy`{`EJ`z)@S74IO`$NfEFf4N+J_lI@F&p7dsTjMCt|@~d-g2f|QUGUvQYAlEx+!iG z(j|P1(4Qd1Vl33=^g^=mYfq)q1l#j-Z$ELd#9E(5wH@zqUzM`|G&$XJ?zd%QBe&}w zTV1Uk%jGZid4!9NDWg8f5It^F0;T?S!f3cbndiwrlC;PSh-^&k(P-3zO|KSFchO1M zHh2?H`2iq8T5R@_R8@cx(qL37^P>lR;8Ci<4@{#5l5s08Cc+Xt#(+K)IOGgv9E3mt zE+V*9ZgMCQtb=5@`&<7s{;*2a9R#cE?a=_+Tn@ufRJn?MO7!>RxO4F7(s8)LdFzzr zV1}U&gab8kN&>SUr+=bpxM$nuTtA#5wC2>)j-5Y1J8Yv9xr!?k!q%= zo4Jb*u#;FuqVG~^bkuraOb8mD{kTC~BBJfBeQ2hhtDYz!&I}QcI9#5G0q9z2tN|Kn z7GSqc!1raR$su7vA#S_x*cgVgskb3GyA|eorXLoPUN38LAZ0VK!WH87{0N!k@JLmM zQ(5x0iiu2Tx9BAtEZ->`?62i0&Pq!3c;7@J6z2-bPbMphe*b?Ronv?%T^EL9+h$`M zjcwajgA+EkZQE+xhK+4Dwrw_PzInf&b6p2>X3y-|v(|IphuM+zvu3ac_l8`H&~+V& z!@0}6E3tHs>Sgo&FQej{TT5*!*mKQwKuNj-NishqMDyHxL3tC@kuiV3Rm+JI|93wv zHvw9wS7IQD6kjFiZB3U6Pjf{a0!!!+w>wVM5+P0HXUbZ71OyWAfU#hv@6zkg)!bPfV25IR4voa>@=vPlPL}{>kavu2 ztOX4njw%jLpuZ!Sq`NPG^c`B7g?}C^=xU|0nw)TvWqHaiZNL z%i~D}M_=_^7D8U+iW%ThAbzvaG7LT_9O>wpw;*4tI>7ixW3xzLcE2TSvm4LfQMsPJ z<~!J&u7zln+Bow&Ux4{_xqanwG7Pb$!}PquJlfo4M1x>b+eR_rN4ov8t?-&M;iR&D zpoTFN^hFPGIL0&mVi63CtNcF~0D^_|Tv3R0+pPgk$uO`8g{hG|hqYb2`ED!7km)3` zfmZe$?oQZ#Pm`s7R0jTeuAB#uktrj}@UTY16f96(GB3-G@;(IbcZ8D`)3?dD)Jz|i z?WH4wgLm`XnQz^*DMsENWW;;{3WLf3-d*=-)68+;vb7xPRnV&V#|q z_*3VvEMn_gXt=!CnBiFM1A~p@lKL|$x&x}~d-S`d3iy0dR+(JzPt{lxy>i#Up~>(;VDdhcl`& zXH*#u^vPjgmuf20uZCc_oiw)oKZ@VNP7Qc*r}$Hz9xmja8sbMHXr@z6H(QNL1v^oi z#qt2x%-_?J@30FM+wb&~j1Oy;Yg2MwP>G6E7+lougKif*e<~D8n=49I+rlc124JbK zp^Y34#*UQ@J08u~-iU}dNU0gpL;fny0nn_aCG=Vv5ec*i{iluv+4PIb-dZ20NGKw1 zuuIaA!YvXM%QfZ6|D|#xF+Op^W;r6?es|FUX38bthHhR zBsa4hJ#9;;#|2fok+R^Qj2|5x3#s^+s~dHRnWW}akiX%?K7}AF^((d*G4vT9%?lzm zVJKKUeYc@AGd_Y`@W$PE!9p#Ox<IOPQ41=8lfAsAPcoNw4+ZB>O)Y|uM~us{PUlz zkzB%kMKRnV8%=nv4Tkm(wFZ$7N=isHmuZ5(nNIWP3p`cyv3O9P5RmvcKoASvu;4)3 zADzn+ej3p2l{jDjOUS>RixaXAzv~Y~?>Zy1q7T2qJsggEC3Uyg)&25C@NYLYQrYeY z)5$8XaRs;H`0UGbg!*6DiG}<6*g_a0)KmP)?+lt%K;GD4q@+1ns23`=U1YtrDfA0ts{dhELvrv7{r7xddz?;DHze4n%VImO^JcXV zOt#msH|)J?y~mu{Hf3wz{_*AXndcj&&t`8S*%|@sfuY@B8SO16k3gKd6>)%qN{(`{ z&|v&&zn(?2;-E{=-<#QNT0ib|j&du(%+e@`o`WJR`bZ!+%8`;i1_AQbGh96hk`7Cy=A~=YbRLOVp74g6tv0O}m zOhsiA!>m+DDKffYwZSDVTH~iTwi4uCiKJ<~XuJtbDDlbV*I(!`Q0EM+1et~8#H{xX zR9edC!-A$gr@AZ0zflxLj zacr?oW3fiZ#qT?$@{?qwIlgQGe%PB>FzImC)i8I|NH#b+td*ODFfe0z`Zl=-O}_62 zm%>OdXTkH>{w=*wg%=UQtGX&5QbMf6siZ=_Z5L({M$kB7Kz?)r)|khGY3l2FRQiWA%}=uXE7F&gy+aZ1L(k0nIt;(w8$dN}t> z!jJFa^W&|U?5~<2)e7#qLDdEW@&L0%WHTxX9tb zoQq{Ph$l0L6YvNQODJd*I)M<>9=VhvbEE`ztOd1`x?qqFh?pG_o97!S;dvs*O)lFM zbjexKjh&k|GYoUEkZEmL)`sbRNHgSjQeuFoenHXO(@ z*TGMT&cNP><9n-AEs-z?TY~ddB^RMFL#f*xF_{1TjOXNA3THw>9|*pOH3a0XA|ctRNF0bg=Y@^*!aIn+)gREc!;9uM^R7DdG>UU-#9bL- zXmm9zkPf~~*Y1%ibqs|feO>wZKxTkQ3)iajF@RRTRaC!)_u9i$IIRv$Rm^2nMhNeq zPN7K^NsiacpD2Bq+KB2lB-Fa&(x_xy;NfEr9vb^>R&@3MEwCJ z8JL9Y*-I^sht->l@yGb)SBwvK;`TzY9j1a=<9FC}w-uHU5RK2(skhtU46i1aM!t;p#XGae~@JldMt*`-j~?w9rMO z3u8Ce?3(B3lXg9nS9N+sG1TD*uubGHIDOk@d=fl5H-we2guoBVvrth1{h$cnI;ijc zgCvM;o)Iy=kuS46X#B)iNRSFo&>0|4JZ#Ggu1=1~E!8yw$VSC#khzb{GKtu!+qt05`dcj>|hw^kn&gWC4X-UZ~24bBU8Jf!fWn~})HnP?; zTLO+;r09Ei%h`kJVaWYoG>42p(CDgr1PI_)rhM~Oe zzv!;JWgsDg)-iOn+bNPERU^bmVl^B}p}I=1Y0V5#qw#j_aFW_l=w2~yDv_3H8YY89 z5e9vZZk4-JBZlsH%DXinucK%j&L*H&*^=r9H6j-hXL23Sspb{jWrUPczrBY zz(oG4a&!NAD8CEyil)&6a$^40ElmK?LowF6E%WPx>sTHoe$h@G3Ixx8&8q{%UTsGc z3}H(TBQD#j@(U0)_9-=95X=c(oZ#`ahtT|Ac43{DP#{urgYj6>JK@S`cC4qexth-L ztW+yN0?k8BD|h@4VRkz+p%R0^bwAFjTKa^%hUHG8HyW zOa=~QyZL%OaJM$slvu#Sod{YGNz4xHL|aZ%gLX=LeJ_X@wN;cvLo)mCdinMhjDk5@ zxiOKTj>cA3NCcuU+s^;g9Ak&#kmi}e$)_^{JmW|_ams3yKKcO!>kI;+u%%2{c~TWt{KC~xJu(dXHb4}*(R zKP#U@mK8Pflyx1JJgf|3GTo5x81fD3B1ZMn*wjTczU~6tBy;3?`)s*GjGk2@ruAq#> z1a*Q2$FH9AKJ80hW3>2};(&;o<#=kjFzjU!9khS)!e0k_z#Ap+(;4-xd_bruc8}oYL9Rk%)@EG*HTigEURx>D1!54 zEK|9+_<3KUM#8#Ib=V0)#~%3PYTp!6!pbQ3*-TD&LV}x{zkWb64wFneR$%Dst(A<^ zLE3((eB+;hrNk$7^F|HvM!v|o9B&px`fJV5V88;&CSsxzIJGw-$8?$}9Ys%87wkpv zE4aI&2bDlgavDgR8v>EL6UbOQUb`c)%R!ud{6r5TAR|%^4%*?>rCnXfiRxUd!Zr5I z%J)8_%ZUH!{xiOlT{z`+y>+G0CZ5R5`4kRT2_Mc%GYkRO1xmcaiyWr#%Rn+|^qpFn zT-1o+A?++?ChXb{JU9(YzXC3^Lhh;L2Hmgm$Zm0q2o6{C<^7gB)kmT8JmPufZDS4c|4Qk zokHzX&y3(49{49h3Sk`zLarCHqB23->|2r-{6!J^$=EgGX)M5G<9$FN=t69)v>KwR zO`ZRAcJfpT1DPpXZ8)WpYQ~LR-vpQKr87VqNk6?J(hHlVn+5l7Df$^Ql?IZDdLG8i ziu|d%&_0g!L{6J!np!$FP$zKf(mai{L5*Z z8Q}@&+Loh4;->b~rLO!y$zbADhq{iPFY*$-cgI}7gYvp12tPaId>@3P_KQblyh-otQ>{v&w3j&=2Z*`Lu-@LM*2+Y{} zkgZF({{5Vv`9#n2WI}-!a|ZhUAH&dT^1hp00(g`tT!P{Pq+s0@Sl5y;8Wrh;%6{cU z=}Ms)M0$hv$ls3Q->{LCRSiN+eE}jX6geFrpMsDe2aBz`Dmc0Mh3?IO51ah@Yn~(1 z^bdfF?l|MnoOp@#CN)f3T|@`!IXIzgan%HWh-%Xd{irt|iN}tJ?QO5=>+XqWEnyj_ zZ1N+~K5tuiUul7zs5Ko3kJ^{)g~W7DTiv$(>gF}m2aqxO95{|n@TvK%0|sp%axRCp z#mk#(C^D-H@o=P~hK2?#>kBP-D+`M+KMdOE|Kzr~1UT08_96S)fFIXr*Lx&RcqOxY z_m&Ig=0@(ohY~m{?Reqwqjzb;X^qITp9^Op0Gd6E>RkmgcCzB3XzY|Mvj`O8(u{nO zz{!*0B27>R_-_T5J_OT#N82_Nulp&U6v!k~XAm-KQi7grrgQyUSk3F!hK?tEuN-nd zL<8zT3WNaa^=4-Vzx&yj2xl*~bvOO_|6G2hEZqw&+-P)Yr6(sg&R}p!a@SkG%aqd6 z%aLWEZxk=$JMCASTFi!`CCbOd$Ontv&&yJ*U+>l1qtoEKRljo2{k>A7zt~`1k61fuopZk%=AB z8{3N8NmFIm$&F0Rui;%WFVJ426J?SRO;M^s!ti`Tuk@32uloJ<9`nh}(_5tmE>o$d z5NjV)Ql~|0^2LOcSG^P%BNytJ@#`w>lRv#LjdJ|n50#t?{FFOpc{u{5WKJ##K zRSsQNV6Pl!L*X$?6}NyE_g#ZDGy`>Q;m)hp50O-iS0K^JX){Y^x7t)jzlW`Tf&>~y zP^dx&UBN)$m98b+0}*LN^#@MXT+YS50vsm2S%6=ISHc74@9QEf3KX>TvE&D(2L;Z; z{gDKkTx)+IXLOpz!-mR*1j+-4fs@i{1DQslAS0)R)a?|EH7JypWxDlGDw!W zYc&YFQ6i0+&1N3HejBp6B`2}xv6F>F_U>9XBsG;fYvDfw`f>%ZEpRn-wRMxWbZN`C zTXEn_{dZTlDS!6Ah~`ehG;hY5k=RB_O$p3nbbg0fXKd>CqGOHpJT1$Bs^q(zwwo&x z#QsnGcs8nIk2}o@M67@0u7f>ac>l3RcjnJoXEm|HN}*OPHusfa&x8-=n{vRm6Cnu! z{{eZMSp@n`?(=j(egeH?v6VtCq}_jYlT62KiC&Xy@^ zTl&6b0}u1J#ck-Hug3-OR;j=nJP}~9FoU3p1(15hCN8Xo)z_-lcS!)|XpcmooDi$6 zEzo9Xf~6-l%dkVUmSG;RArk!OTHx_9@mG)0ZsJH?_fg;f=#H3SPHyh>o#nGQo-MLm z+d9CQQ(;~RxH$G$Y8*E^%uaSsFPAP|k%1%B7G94u6_>XbhJcMhs)E0UX4pO)h=|u4 zq|f^s{a>?*U7h$*U!Yd8&zMd;0he6u4>Amq*`p34c2GxFa8Nw)*XZwNo{`-yhtUo^ z-BJnvO^awO0R0ag@b|Knj-Fml4-wXPyrhPPi&Q}oHV)hesd7vSAaqZF>%)Z`xr=o_ zke;}U5K=GnaCe_4;Gb_}S&#Q4{jM9ovL%T~$sPOoF74Ff)V%~XN?FGpKN9NrO6Pwh z(?>VZhE?T2LnV>OwOej?wGPHzOH4#dup$xJ?+bwy#LM|Ey?|Bp)UstZRJ#_wd3Li3 zV~+OUHz`T#9ZBG^=TP zD5IlxqW3|K#l!wd+QxodnsB-eCPE+Wj@Z}UGfJ(m?|6zCYGK1)!hC%{XgN7!tCuUq zm7hGvcfTGk5?<4gDi;QY7?s#I_Y_amzTb_VmB7-&k847upS99PT6ShI8x^C6jLcT0 zR8#^bCi;-F^B_pr|JH7M{avCVFnXqr@Zb1TT1~);m2!t8HQ@}_0l+zLXKXbMY<*5lIgHEIhL_&I33|-#+D8IrvR;97W<&as?Bdk?A^m~8=|f^XuzN7fAe|>b zlhD);LutEQmv@M=lT)cv#s1U37M06UEw8^0lY>ezjggWyD|Uyku`6-k+^$lrtQyPT zk!)oK z*^(Zg@U?UHiV15F)X`V|tKxMPnudf4Mup_M1wkv0hv@R5NM}W*Y0~aI*1S|-P?<43`_%g zzQYz~|KY+eJHYuvol;!_xDRMx3RPeQgQRV9Eh3xuox^LY#RtpIFy_N1(Tq<+Cg+2+l#7u@ zAQMtc<9_*rtoL`AoiKjBO!&6aJ$zS#HPOxXpP}j)aE95w-0a#1<4N(3DRLxqnyt9g zsP_hXe;WJ0THa=aIYCPAD`l}uY8}5QO(G4ZJa?7O4k_R|D2Aq;cI1%|%T+&cEWZJB z@PAC^AP*WhL3P4NuJkrwv{J+iq($5Y?;lN~v`mlz3bHmQvqgJJyE#tXN#fO5N=*&_ z23L`E50{o({=385hG++6CnG-sce(=u)ZCTufg-e2A{~+k(zu{bEXcF3y>;Hg!orQS z|G|FtZwUa5vchVv6KaQJCN4v;vEoKS*ZT+Q3-sWI7pHyGOF<_07m<*!PXJUk4=iuQ zc^m(mZsURm@m79*!#x~Ji2#LyEBH`SP*_+7@8?tiOmdez6-GU(M*%Q0Z*-4%pkcl} za-dwU6Po*0ZtW?{F~_srxz=ER0nz&&3{;>q?G`IVzx}B#I$NsgL{K_j#%p6>1%oN> z2Nv#w);I<|`>kQmG4>lT8FEVOqRmt%Ic|M{fT_fafSDP83TttrEOl?n>uP69^Cnt# z2Z67)r>CH+;mA`vJLK(Sg_8%T8=MzdoCW~`BJmg;rey3$h6Xny;9$T*-o!v>2&K>{ zmq%kW%mWy^We9mL3t&hnAZBC6a66dN<~91B$$v50{X+v=F&>6GUfpc~?3$Lz>(+hm zZkiwquX|T+Ko_6p^odM;)G-P8iRJX79{++a2XB?s!;s3z1~~R4or$i$NnY_89ctDU zG`sN67wU^=%SmgxkZyBw*lW~r-;yV&ZmHPS+#S7v0&;jhsyHvkcmu8|99AEAHenWE278AF4Y`{z($@ae>Zqp8dyU&veo7E=JTlGC~dYy>!q z0HBo7?`VLFOC2d=f*8*lVc^V8{&0VvM>{G=s>AGk$A~_7qQv8Yft59LB4pN;|31)0 z$OfwgHN|pPB30Um)RFKka`z)8nj3-!Jjg#8);2SMLwElHFN-q-9L5^8B@H@vl=f+6 zq($9^1EHO*G&rbGHl6_d2wh|(p2Qk8i^r2v1!b+#FN3!KY_VRFN)G4vo~@}7vV|2e z4|{uj_s@1Y9#f3Fs-Ia#6Z*xKy$K7uP=#4xdq4cc?b42x%SI!ke}IxUAjlU8)H$#H zIwXY%p))K`Wu;=J?xi=bB08MPklx@g5ONQnL0FM!#p`^=Vv%|z_(Lwtiq(?$`T5h4 z<8r;Tz&KR|&|{DRvez^hkbb>UJM|-T_Ge9@*Ibt|m2#S0S|w;sOkf3Gfw{~Xta)Hv z9ROA7eT?z{1FdXl3q|QHmv8@bzkA@R_y{as;O`mxif5*N+)Ypj#hxivYX((Hr5Gus zwzG~VFM!Rtff=^@{-=~4-*$E8_qAamUTMt%jCqoW3|b}jt1fWv>%U!TK3aXghD?bq zed8(gnMgHVuS5=oud(2nZ4mrG@mW+ndW5^7i&ahrs)da`ZE*k@7> zM6lF5DkNmfiPiK14qzkkL80 z6NmRlEM%Bgxl7KWsE+HuEPsOv0IXYeiQtjAL`nla^qZ88x?>MA&J8^^DZEpqMpO0YSj*7uj@@5HFz0&WVxj!;E zB&2dPB=5hdm~Q(K%T4c6!O3jOL|$sz2#^o2vU(kMySQOxa}@gCixXUFo7F^y}#U2;}eZ zv_nWEeA-T%ALc5@i+~AE1y7~f>T$DA`wu|3iR!o7m&-@6iAQ)wxayO8=8nt=BN}$Q z<#)K(Du)>P2LF5Yfy@)|$b=Ev&SEy?9Z8WX*6d*^o<`No7(32d7^(VzJhZ*e&#T3J zOnLLqyrY}S|A&ovPL}^ALTy?43{a8>+ze)%-1G%hx#pZbxNJ+G^uKa2j=jVIuG*p4 zLo^OZ;c`lJ&i9`cr)5G3H`V0PfZkcz3FV4mlAM;8LD@c_g4yBQY^-g;!YT)it;*-f z=?~Knoi~h$hlzr9@5nXoPUTGHrwcxt9mItBos15-uP$LPbA2B^Rm#h-r(_(G!omS( zbM0(Qf8Br8F8cbuTR@kB&Ts6Z$T7>zp!XW|WJ5FIxHsVHQM`jl?3Rt(xodvR zQ(zUbpm%CG3L%C2sqlXnYSlpCTQ{7{E<8s(a#~0z`Ai~Kzo@8o>h@flrFQ=O4LUZRN~g2Z_oBJ zd#`5?ZL3wA$$r`qO0Q?Z2WRzZu4otZ$mivsh>LF%j2~m5r!ng%b+}hbbDeXRCzr`J z9Yjdl-ae>-l|9gq6ivTG{>6ubIM02|C!5e ze;8v+Aw&7@YTGApI#*a;*#8;xqx&B&N$iE&=^J*nR!MZ}cCHAtx0r4=_r0{ya9E5Ek{1d`3hfw03$Br0y2sT^28qLRUWePI%rlRJ%#=PC z<}J3-d@ZAI=%rVFSG(6AbP1mRe&|y>oYFJ-(`X#gMrfWTJGy)|BjS2M25o{283kV; zx?Ag?QVm0`NTJp5x?EvoqRKlrgs7^K{A|(a7WGf>Gr>ZzEdW)X6Xmv=0``Y)d!{zo z`s(4VJG}}kjbeUx?tjYRc#_qTl= zA7^a{xOeE@Zkn?+9`ezQBsa|Fw2TM@9qEz1%DxKDl@KoHg-PCHS7~jIE3`(Ieeo(2 z-1U8emvDbh19B&iUUAur^b$_-`7&`iKg+|PdCcDAHb{g99d8x#-@9d8tv>n{n4*>a zx+o3-a7m_2E~8~wK!qZ|mfw7Gx?(}?_oJ}XZv&>!2uJW}TiQ-Gp9GU~2qk2gh1<_d zLqs=?C2YZWGjTF_*%$64VRDgD>}<0jFlef%qW1u3Dw8r92IwNvPMsK!RSa86geJIR zLo-!e2VE7$#cHycrMqw6u&QWVpzYubLPG<^F(?wR{I)hGp^~$B$jK6YQnI{Uw?D3U zoCQCeYuT7kz~0s}UV8fVN7b zC}CH1huLPO)WR7=b|uN1JwEJKj4=)%pv1RReJmFU>&JGAAgO%jIFc8;6S}&cYY3LJ zGR~?gR>X8*9oRQQMp7s4`+_08A@Q~lSzVo_Wcuz_?s33lL}=k=Z=sP0?Z8A;6{mWS zlV$_`PUo%hY|{6P${OE;^GnF{eAN>JFgQZ4$G-)vAR^1)c`6*7v!3*He-yf%EGk+l z%4ZzJ$9;LeZdYmj3OSPApsgaPsi%fAy-&7Wq{|P^HhU$?rw-$|ZykdZp9so^e&q3t zxGhClQ^C`LX>PUOL})IcYVw_;u6%fwS?is?vwu=%+i~aZF<-VQ4f5geyfL`d#oHLL ze7z@JQgUr%ym^w-o6~eM;&aigz5bRrBO;Rzm7lNVu=Z08l`ftL4s8Fm3*@afmzn=K zt6>}56Z2wD`&*-$1CE6B?ZucCUy#On#`1k?QgPUS0QNXe3MzP4f?^%*k;;^z%qli_26ZN7wp~d)$^ucfO zPSaB5S$yd)R(x*vF-Qriooix=m8=X4LN=D7+n~mmC`Jf!auFl&vQVtd zEwm04gucZ0>86tpENtSE!!pbG5hZlf;KphMnAy1G4g`WkZ)1T3!7}4yU+b;s!>Cmb z3)`mzi;tt$ML1)X#QJ=LMrTw08ajcLdP^4-pMYWBNta);Z;_@A*aPZV?pK?S@ksK7 zKOP&1%aI>u*&VnNFjxkahS=Dg+1=l0@p-&beQ&lXWR))|AR{bf?lQU|)<Z z2u(UOlV8GIQeX)9DK(R^L+^Uj^w2)UF5sSOzx`Wsg7>b#{BACSbO41z*IHaRrJE@v zqD<~jYk6hwZixkce^FhBjuuLlA8r>e|HP!NYwnXNx68$7fTia!tAgf$yOtSqtyS&Q zRAWdR`;}(Swv-+)0zqghyXf|j7usM^n2V*=#8QMA0Ya@6#iGzKQ53&TaZgi{>6@EK zQitS?gUuCryZE*Tvs1L#q4cYA`E_Z<|5A;zXxBx2KT#n@E1=^o+frgok%z_m(>E(* zwTCnb5q;2Di!^5R3*`)yr0a@P9n{SuP;sS*Pxja2GH~3#-G1w-2w`BMV#lxJFR?X0 zxFy2+Xs<3t*c2&GH${&B-{e|P=M-+5luAX7rm zFFhl}PBD~#hdN)#wV2wh`?Q_Yw4t3K#LM6j8gUW(jCSG4^cHl?LN0XK;2O~M+K;9S zT;XA}F(|sgia%!=+lhl$f*Qz6Ol-PVlW#!;+p;+=#1#pdrHn zUu4@Zutn8I&J;q|qZg=H+HCzcid-$zzI4RF&T$KA{k*y1+S`_^-K-y z4FO@EQSjW79R8k$4m#k#s2NbiJKEc#AvJ_TJ@j+YaKOLZTQ~O-Nj)lZD0SWs$6LbP zQ$_Nk5ABA#W=i&ukbr!>|0CRoc}C}Dx~Xp`Os?&5ky`oNA}?;I5EJK*Ou=_GC_-C% z0|rA_Z6lORB#omCwuQo}jCYyrLwe6O)jaQ_*k_l6s4nHgtLMP=wZhmx%uX;6?JXM0 zg>dD$glAY*dT{Zvpx^_MG|KMo?%Mzv80X5pz>=|nr%VV&SP+K~DE7d>kOcGm7=Mi2siCLlNs)eWLbXGtLW8>GdH+T@9VkZ>hyySe$W0^JHhy~8(@dX& z0=KOq(CON5!c!S*I)l?G5zYBDhPVb03(oMFj-c{Bw?mWS?t;GBUed?)m8R$%M?0Y> zl&{ti95@0C&8Ow&9&MdvY@G{B$Xf0t1y6XGY9*J1&mP)xy$-P>%!H%T`+{MLzG4Dew<9G6cU9;l7jg1NW@@-aI1qz|lrp{8*bjj&faw^K z1kBWGiD?s~b=1Q}S}AUxWzy{HUC+(r$lJ>pDeS~f zqwIjIz)_4HheIScQeCG#&DkCwW-KmVPcu>XMuf_I2g_1<3veXrP{A?Sq#uV_9b|r< z{WgXW?f;vfYZALUEvQI-vs$6;ipVF=x_(mkgo5Gc%w~m|%1j&jZWbCk;3xEtEqBOh z?lB!{5PHTSKuxvpvOouSBckhFQron`L9=}-b*4Kl<1eR>(AAyu`Xl+tgdrmMULAV3 z)L;4lkBdu%TV7V=^bNXXv_epBcgll ztgEwns3dK7Md<8tucL-O3KR4;Of*m%IL;|pA38G%ImncAYy%DI{@zq8_Wn3)J~`$F zO>ut6=YYUYh@e_;YD6$Rgn;|YBZcrko%kr41XTsLX(-AQ1pJT{MDtUHra^`K&>S-1tUFG1^w_G%EXFroP$E%LsfAZPK zO7>EKzLA3{oeYdu!)rhOUXdQpP!Ywj&wA{VUJt(Yd_DY|u^+%a>%1h;?V(jbv+{c8 znzdrmZis-!La%4nK%uzg{&(!R32oOTboKhYlZ0&nsuE8z%`b(R-XBq5o~JmSQ+2&K zebml4Abr@3whm0PG@YA3mW;Ax^5E75Xy=_hihjABGc=dg9 z$A?5M*B{9|jW^)Yv@U2=ooLv<2kCB7fuBSp_l-<=CN3F5|N0UWDA`Na>!av}f{sG7 z+Zn|gby4911!qJRIGFsD)!UZ_R)n6zGIALA_S<9#rtU#WP`;O;5yB`R@%VbNZp~j8 zO6WV3jJN?ukiO#eSME_4^-GqRU6zi2{JzThsEkP=2xwPKLrl{p4K!FW?D^_ip)br> z&Z?EpB0H1~H{m4s8!l`3_^gzqqb^B$$YsqzG z=D1QVi(hSnr?I+>C!OOyShKnc%862pCZ4s#P8}eYGgD1PRbnz}v%Wgz3sKGGHW&Q& zB(O==U^;MZc^uz*I~gm3woO&v)w^lyID1|->|?y#0r&a-Z=??aiHd0gfy>YRu^9LX>6;aF0Vn5RvOXrXd#EW5o2jld0VMS(Qd+ z5Z5gBxl#+M@LSy}pBu|G`2MrqB+2QRtXTJTTzJhm5Ab#HuQPYXAvf}w79K`G3*fwH z1WqOs^ai*#slqcU&AR5*P+)(7GlfEB8@&OdwEGjn{*wvfzu&<0-*$TMs{Q|8P}ogr z0xjeB3BB@9A6RPO1nE%#+e|^ttH1{EYCq{-gv7Uhhz%dDzzoC)@lyiW)`PHa{%>J| g6Z96}`w1SZfS4{Z*uVt*oiLcJq>@CPm`U*e06ryfDF6Tf literal 0 HcmV?d00001 diff --git a/docs/source/scheduler.md b/docs/source/scheduler.md new file mode 100644 index 0000000000..209fb1d5f0 --- /dev/null +++ b/docs/source/scheduler.md @@ -0,0 +1,55 @@ + + +## Serial or Concurrent Inferences + +Schedulers are special system software which handle the distribution of work across cores in parallel computation. The goal of a good scheduler is to ensure that while work is available, cores aren’t sitting idle. On the contrary, as long as parallel tasks are available, all cores should be kept busy. + +In most use cases, the default scheduler is the preferred choice when running inferences with the DeepSparse Engine. It's highly optimized for minimum per-request latency, using all of the system's resources provided to it on every request it gets. Often, particularly when working with large batch sizes, the scheduler is able to distribute the workload of a single request across as many cores as it's provided. + +![Single-stream scheduling diagram](single-stream.png) + +_Single stream scheduling; requests execute serially by default_ + +However, there are circumstances in which more cores does not imply better performance. If the computation can't be divided up to produce enough parallelism (while maximizing use of the CPU cache), then adding more cores simply adds more compute power with little to apply it to. + +An alternative, "multi-stream" scheduler is provided with the software. In cases where parallelism is low, sending multiple requests simultaneously can more adequately saturate the available cores. In other words, if speedup can't be achieved by adding more cores, then perhaps speedup can be achieved by adding more work. + +If increasing core count doesn't decrease latency, that's a strong indicator that parallelism is low in your particular model/batch-size combination. It may be that total throughput can be increased by making more requests simultaneously. Using the [deepsparse.engine.Scheduler API](https://docs.neuralmagic.com/deepsparse/api/deepsparse.html), the multi-stream scheduler can be selected, and requests made by multiple Python threads will be handled concurrently. + +![Multi-stream scheduling diagram](multi-stream.png) + +_Multi-stream scheduling; requests execute in parallel and may utilize hardware resources better_ + +Whereas the default scheduler will queue up requests made simultaneously and handle them serially, the multi-stream scheduler maintains a set of dropboxes where requests may be deposited and the requesting threads can wait. These dropboxes allow workers to find work from multiple sources when work from a single source would otherwise be scarce, maximizing throughput. When a request is complete, the requesting thread is awakened and returns the results to the caller. + +The most common use cases for the multi-stream scheduler are where parallelism is low with respect to core count, and where requests need to be made asynchronously without time to batch them. Implementing a model server may fit such a scenario and be ideal for using multi-stream scheduling. + +Depending on your engine execution strategy, enable one of these options by running: + +```python +engine = compile_model(model_path, batch_size, num_cores, num_sockets, "single_stream") +``` + +or + +```python +engine = compile_model(model_path, batch_size, num_cores, num_sockets, "multi_stream") +``` + +or pass in the enum value directly, since` "multi_stream" == Scheduler.multi_stream` + +By default, the scheduler will map to a single stream. diff --git a/docs/source/single-stream.png b/docs/source/single-stream.png new file mode 100644 index 0000000000000000000000000000000000000000..c324a2045d7103401bc87db2258e3937ce28982e GIT binary patch literal 17490 zcmZ^~1y~%<(l3l_fDqgX?yifwyE_C6?(V^YLvVKp?(Xgq+--4pmv3|a=e+ma_xbMh zGc&c_RozwHHN7+4^$Sy!mq0?mMF0Z>Lz0pdRR#kCj|I`%a4?{2QX5z)C?noNL_|?a zM1)w;(ay}m+7t|oD$XZ%Oa?*;eNceNC}<`xC+9~%d1ARP~GW`aJ2(0NgAD7F!9&GyJN0P+42T>Km zN{i4K(NL-mX@E#O`cX+F8>V!r-h-Moes!PYmlL%;8q8<08a z?SL*-k0?7yWWc>HvO70<`mEp-`*efmq#V=)QA?lTP`pulG+3ogjy`+j6lvdYB9$9b zMHShLaxJbFSmhJJSA?ZiSaEVsrz4^m03M+he|<%b_bY@Gs@tNtV*~|glxj^irOf2y zz-T};92hM47cfW=1rADJ;J9GW|DwUbzJcTY4{Zrf^=}>s5DzmL=n94eIvv4T{zapM zX#Y?PFsOgaCV=jL4@uAos`a0{gp`~jF$)7T69Y3B7#Ir^3p0R;4Zy-h%*+m8V*)TS zgMmThK>Vi;@Yo#4|IlFCe=8d^OHu;e;O!-~oWQ^k(f*#`U}+iHf8{JxHJvr(WC6x@ zHVj54cHc}H+->atN`djZ13;vWsk0HWyN$K26TqF1B5dAlsk%ahPEY4PZB$^-< zMeH0+iP;(07??=-5r~P2c^ys60Lr4`{}u=R;v@O)>}(HUWOQ?LV{l_-uyZtLWaj4P zW@KVvWMQEPanL(?*g6}z)7v_c{!__+>Jc?{GIq4Eceb#zCH`Bl(KkC6XFd{=zYYD* z@sFO)7H0paC0nO|n*}nE@oxzuGXoRj|DTwtyT$)Q>~G2cQw-Ec|6|tw)tI}H{Xatg z>WqJMD;MN3{0Jb)|9BriLd~rH zI2aiJisF8;K844Y9It{|rY}75!XCrKFhW=W|zNwu#JIR~HA}YS{SVqM_locv5*>iCi_aiSUKk z{y!<_C;o3;(N}*$uFwBYhWKy@WNbm*0so|L4#ZTk-)iW|5b)vtNeE>1|1Rqe_~u~P z@vRUc3<4GcR-G2McO!Xi$)H5JaP2dsdJWfSbjg|}%rDT$<@(4_JiZ)QCHIEs5z&oPr@2ekZY_&~@Z4xx>W;@ofoLdf)}B}4!Xc=*=#FVfD(YbQ$Uhx< zst;%Z4~Z|}Nt4nQ%B{~^=%BQv!%ou`^=;MQsf(|E*QSK%@elCfkuehNa3n8tzKCcP9I&#GJ@n zRl;6lQ3hfI(q)2&m*av-40@Jkp*bxPqYM5-2mUb`%tqTNiY&-M?DyjEHthv*8KGmgzf-kC1MAms7*h8 z0Y6tx!t-}$TYxu8pNk|?nO0rV*}YI){}sN#`5+MZ&LzJ_G}J@n7Y*E2z{*AvVE1uH z{+C<*9>%ANgAl@gHY&25FC}mK?imbkEK@xN$Lsxex5%lBss480oKbYa_LZJ3ClMj& z$|?5~lKrOdi;Y&ZeXeSiJ|DYOfceb8(a=qF^3d4fBfe}N^1G}z4gVC_?NwmR80!%P z6!PCxeuPie2q6S_4XmexaM@`0a5aPKLLBFI+9!kNZtnkj>GyKRsa0ztr5sD5nj1(A zW4ByuO7Y!896smM{$yr4dhc}TG*o?X1G}qJf_&`qD0>V0PcH zY054PT6MB9u;@?+?8&jd`+WnyxA`n;Siw|q`|d*if`eDcsjk=Sh!(0}Iiitj+f$ZX zU}vm?TY?r2IZGAr+dqlB-k(qQ5>({l@a)=^TsM8(QrWGN;GcOe+Gls(Z?ZpzK|aHn zlK8n)!@#FO*S-hZag4rc$wC|Iq(CZu_x}Wcq zm2z8tw$H=58;kW)^|hJGFrx%=0iR@wu{PR1sd!|p-z$))2)#%ln<3q{wWJ-?M=UPK zs%Ven92^_&#%*l}9S<9IWpt7^Zx5SerpbC{5Lu}ZtT0e~y>Dezoh;=24OpCO_JNUj zoRim<%e95h7aMv2*VFI9ud@mQiO2+i;UIW4>&BiAPAa0quhmL|V3{zjU<^}KF?L;V z>i5g$L@J&-wk?I9u$j8eFt@xx^HhjJYKl}YT`Kz~>(;h+4y z)4HifHvC=_ambuuMTQQqha?F?kA8G|X%y{$Oh&@vNJ!TA99enod^y+ISU$+uq^)ZI zmIU<$-*w5zwRk?7=RC#f*{G}Yna=#fvKPyQUNEzj6R>vW>AmQiBK9oJs5H#YzxejU#+FbOsCJXC*$& z#bz>{uh3caO@Mcbr$sTRu0uIgNavWCvF&`?PrCm}8qKr^$v+AQmW(6b;d!?kO61D~ zJscV2sFDrp?9~LrV)@LB_A7|Ouxs!ARPzG!sjO>|N2T=&w>Xfb*S4h6rZTyM1~(QG zh4<1Q-}|zM!+pi`q+%}+g}<6MVw7o&B);>wv~2eg7}NBQsnFhs%meqJ{nd&eGXD!aoqKs3sQk z>%?~==c^2<;MB_oBb?}+y9ib8y zk-nec@6n+2hfZBbqIy2eYi8OZFSPW?>l`AC)0`jWPXUl-*Yv19qvG_Pr2?Wzo zsI5pM0sTpV4{ugI^IpX5y`g|_RRPURzRyR+HsCl$!zf>uo9qm@wX4nsMf)KhQ<#nE zGT31-n^UEx4LAKhe9$%~(m9i57+U%-mxY4E;0osoWw5NMBDpjRLXfvp+M;={CDZp} z8=Vd&i&PEqRBpHXLJ?0s8xKV{;D~rn1UTlyDY~ZgG6^YRnvqfg(BW8fI5*t;7DO~> zg_LsUzIU(v2!{s)pi#=#mtCZyGDjHk1rN9Y$QpIbgkcHYb9nEUjEg=gxo7*mw@_0g zNd7f_7wj#T|FY}j%?@L4`-x??sLM!RU$5Jfg5tt@v~Mh=aHlBUww7HS1@ZhCC|H95 zCnl~{y`b3T>w|U)XEx5NW7Cib?^TqaY~b56@b$4J3M05L@(1e6b|{gp8R57g+%lRx zSC1JK71*W48dWMRk@xh%7e#!kk|>UqVe~%@i~3f=AH9f-3H(0KOLWjilenK|UB<;D zU-g!`({lrad+CA|VUrze^n1!-4UvNE@_3&Q^BaGLhJ`=Z#wr*Op$~ok*?>M%o$6mZ z(|lCjcGj4lIyK4npsS)82#b<*kmWUbnTtkjpHt4=oda`YVbm>~jx$;}#jk^HSu$AH z*wYl6^5^B0FZmO85VH<^^)t7kRIt@dWgyn}=np0p9a+Y4V#QC)Y_#m|VEJB9R01fZ z8F3K8xJFXI?r}&YC4^24WY~ROsLm{?tyl1XH$(bR8Z9j(Pe>$g( zHWz=ig^@Au%`4X@v{zy~y*FHk4&L2!@tg~0QRF|&qoRhi$W`Flx+^?Q2*W{g*ivD* zCR@8{61BP_I)+zp-=tDEv!@H`4R!a@iqKNDRCCIjn8Wd&Et1p7mBAa79TzP3dAa9k zh6r0`Pv>_GnBspKH`{O7N`eY?=?Bf$b&_OE#&zJP!?9nJq5~*+*p{8+O9X1PU|DN# zDkX7Xc7?eWPjTQFgNS}$8o|3``z}!H25d?E{Hy>u6eB3+Q2iJM<{A(cG2@uF_0>L; z0wSHrin@6^eD;AQj?*F+=`$g0mWA`~!Ds*u%05%pNIc6WDI?Z173%R;9eozK?)7!8 z5Y}^$693fLQ9%-=e&ZGjJ{2-7QwKzfycT1B0YbGGYMm8Bccu8GMvbj9rv23b7T#MP z4a)fV9O9nCUZRG!A0C%|4mHZ?(Mf`-(!L-oD{S8U_V`usAThQI`1VhK=+Bp1q7bZp znNVLRqCfYdHf{wXx@+%b_zD3~$$DFn0em4$5AWD_K#ySi#cKD z*ZbSNZVWjVj!a5S>eN6J8uHD#@#Y=k$5MiiW99rZle}^eE`K zUm44C8p)O;FQ-gG*QAU>nRd(_+B&%!B1)!jT7>m88>L)~mDv(Ja_GkR*(RSV$6&3zy&r<5@7{xaU+Y1Jwe<>(xIVy) znp>`!y#*7JNB7IgKZPa#mdkwL31z`pOC%EVnzZ|PPY_j3GbJYvVn~9!P$gJR&ZtnN ziAr@Q`Rj%v@ZCBA_!{<7SEz-zlOb?kiR7ZKavU3VP%pF~w$fll$#Yg8AGNzYQV)9X z!_el#Jk{zpE92uKasqc{cRZ}3>i!f+>!0!jR=#Z*f~e@g9a{(C@b#xpmo&z&+_zc8 zwQ!yFodi1EJ*Gl46Ih~J7ga}leFwomiy{l;`aP777>0ND#5yk~u=uP9$QX_c5n?ut zRWA&^vgBs4WQ$Nn!aIW@vfVufjZmzCR0#G!(ZxzVdU#mq7R=!)5*ma|f1OVJVXc5^ zs?@>S{KG#IMDYTc48KZG)bz@r#R}-0A5z|VPA^5yvKU=;T7C@QmVVBXU0xP`Ou4q~ zpi#qUupfvI+erHyou}JqT{2hEl#B0pnA^Gs01&BQSGC((6Z!01Q5Nd2dycqJ#ptZ+ zx{C=MPnQJh52@5Ed0JfNl}u7C9S!fW)ghRXoI36hw)Ohg>8!nWr&g+NeP(JBLWkzc z8S`;FEr;Ne*$XeAa+bz>2v(Xpp~t$AIa_H^9X#r0jtkJ5a3ALIy>I@0ic=_?Jb2Z9 z8J#^&hZ=Xpoy8CheAfl`$Rrdhm&>}-YLwC;erS?OyaKw7URdizUJO<102#OHEuPzbzFD>U2xY1~#{csBdZaHL3_ z@$qQ8@Nhjbl|3d^oRM=xG7xX$+)Uvp7nL@O0E4B=(W1)0|A3(PdwmsQd+d z>?(M}9}~g^j(WKhmMg4KELIyUCT;K5BBN(W70r4We~M?Z*#@P)T0LGl_iGw{AwSAp z$F)(s&tzTiwPy+X}u!jVyq_G@X{%tYIZf1q&%UHU5d8A;!B0`;1FRaU9nYs=Q+D zywT|wKk$lM9D@lx=U7Ql-KoFC1ar+@D*)B{8H@1}gNlE2y0^dZMR<>UXQ4{C)7YXP zQ#<54dL6y#^*^B%qdoA@&v?o!Xtn3<439#}t6u(FdlN_QmP;N?$HZ(<)K_Feo-J zhwqz=uI#BmuIUVkL{(2$CkwA;DKrp@LHBocmD3H?$)(m$Cw+ zAs{J0>@oc_-R1Cm4_B~vIJA#^>a^)clcl z%CG8Y4;@LB`?=NvFOEV}wmiGueh*H!p^6SDi-an>co}U%F&z2)LF+lpw=n%h?XuK{Z|Y4AjX%iGZAX++ z>q?=}Nf{{LDd3a+4WVVy(A_cWxtX21@du#uz&Q2l2ICMP1H{gO!ZdD@rmUp7D97b}&vOA~e7ENbu&JyV8R zz*&R9%90_;6^XSkFW=QKR&_)i-O%Q6mDd_tQ%)m|5$%k+o4A)O26g}sMh;(ieIHaC zZ|TZkAb5(0Tafq0S*O@;)Ek}q9b&MK-X`g?ThNmj`l|?$(G3Eaigm`!iI+=56`kWn zKgViuo6@hMV1_7>o(>N(M$wCS!8eIG)#Obv)rJwcYF`<#Pky(@kcM36PCCJ$TC1NZ zMg>D=bS#uI>K<`oa5UNPEr-5DB$+LwhtHP&1#-WI*6LF^Hm&s?+E{jn`&?6Te+OY+ z4WXo{n59VW3nAm(WQAE4?K<=Cqm-^$s+C{pEh%+mQF$*#KJW~(3L)zW;L;dzkhcKQnY@C^f|os~a)pD4YTY@mtP z5j3-0Q$uwMk4gYp^r)va1fH7b{J|k3>FO{sZ^phL9GD*Sj8edl))oZ+k*7B|dTbdu z&|cNm@lMli*jgUV`suWNT-m`Dbz`pEZSb8JUts&l9`1olAwDtMW;YsPZ}aRY5U2oM zO%JX7Bb!g+iG=R^;~L9oE?uSPrCr&_Ta&)gMYC3_2^j`y-r1c!CxJryMu9W0Zuir+ zukOywAE^NcZA|1U=Y)3QCML$&iXw;S{wjpYbf~UxaF=1ru*KKh0|srbXPVZXi=)_a zBPg=K8-S~1HnKGGa(g(~tK2Xf*n(ot55s&F8x=7#DqWGH**YeDDr-8>FM$644kIlZ*QDL&S z7ctVRXG>A$S>L$5W;-PvBO~JzUyilQ1ZJ{!5f?*U+H#$FaUQyRG*ANd2(K9E(Pq!7 z$7OV7ZzKCG(0UzeuW(XHGIW`LL5+IgOw#^pKZ;X)akbp5aaI$yQdDJ&j`N}e0iCDX zjxqqZGw-Q`pP!k=ZY4RP&F-RONZ?dLJOs2l!T@(+`?Lsche2?*(0I9msV_47!$7ZU z;PQ_Z2H(}`_Aa!~;asH(kB2dpv!z9r3tJ?S4kGuIQj;yg(_ZqjW1m{np-^R1WM$Xq z@1_uFm+)UFTvhy81YR%uT#cz-md%efcBsl|(Oz4o#oYws(VFMRo_f`Y!FUU7=i*vv zmPOqYP2MC`I-X%TAVOeFN$uRlxw>UZb}FuQJ%IV9j~3hT`VQBG`ZnzjkWZ)a)Yo%A z&F^?_@T{KlQj<`}u7^&L!WW5>QOOUWJ_~!t(qepS`fQ45V+dJ+_ImYvd3x92R@t66 z!Nv4)#UZ@D&F&oxL*|h`t?3td7!j9I$~b&OWq=GOED$bJ&q=P>)#)n$JP&nVJ&|25 zFcPtX&2rPma(vF$Q?l*W6F>DfiLy&DHv8tkvvudDi{iEjT)(?)9`hQ?2nvJW>*-MQ z=E=N^?)4YRDi37R+46CuHfA`=8PoWZ`c&O5zw!5r-7crDu{T)?C$h$U$_w24&xTw! zX38}s8qv=8BYVYZq&vPEz0`|{0+pQm@GlqC@{H%9Q`1%IHa=%<^9>i4>E$YVQ$3;v zN_q-ao~4;^H}B&*Ri$S!_Lo-kMO~HVKA5=dcN*p*E>>z9+*06QSOl+*scecxHip?x zU~!S@lk+d9M};@iDlO+N#Y5dt^hPzy)RE%BM?7)G|9ptrl^q_jiNT!ZGDl8+7VVnK zB4suAwjywy33@1$S)8nOd0<#@7>x4K;Qw$gt+$@nZCvjxW{&H;Jw9B#Lx?3RkZV(4 zNCDW*EAhWys#r8jL0Z%g?63GA49`t<{i@_^#fB7kdvx+pu&}E=8m#S`^g+vx(xV1$ zWDy(qTs6S|+E}$A)nT(GGw^w8SaY3SzaEC{E7S|+m?;-iIjVJ$;F(U$f*#%Im)SM_ z8oOLH9I-=QgMo_2EO6TAos~Sf)M&$FvBEZmE?i)r7ZS^XQo#ygvmdEdE9q7}p2?$7 zBs^ximTKYiQ5Mc%bQ1&w0rD8{=OHi-SNdfg~YfI#! zXVb~_!g)a&`wdSjk9n5!N@-yEG-S42JI8|ibfb^e(Q;c`xoQX+@|3yHOUK;m?@H^{ z;$rK9QJ1wPc*mh>FOxsL11s28I8j9tX-h8TsnF;}_-dB`yg&msO?UA(_rcByxP(8n z=HSnFC+1CkUqZVL+C?Qao*B_ZGaJ&pE5nhD_^tLIuKOMYMkn81q=4;%EDsMY5LM0G zjS5{hg<3a7MHzk{Hqq^`y;ilk$iy7W1-Wc1j;v>z4W`#~Rkv?$%GxI*#Y74<56~#w z!;d>ojH_yF>z>y1nXUC2LSH?jrXW0g$0G)vR=!>EuWIVJzo#behDTJX3^Z$YzL_Q6 z&e+4q2J7t!6ZOc#aVc=O52=0iE=?plfz`#ReY;;>HMg+v3f9Bf>E$FK;v*WDp#qnc zfyKA?zkIvZ<$vtwFZPW`C5&*kIu?xYiW7Ka;OvfTt?=A-URs@S;GWFvs8C;JndV zY}(~~^u^<80BbKfJ3$`h@b|cOt?u4IXi<5QH;2}X2GUVopKVb_(qRs;V$mRBx$Ct6 zaxX;M5B99jMK-B}^W7#fo2cZ}igfjMLrGm-y99FZFkN=}IE}gziuJS2qs21-X%4&W zhjBE*i_bF=Nj!xwvCV2z+z64kb+5_oZw=Emor_WZms1L3eIFTp&v&)?N|)X34?c>i zXm26Ul5W4t4GEK}Ok~B^QkqL5R->Ln&y9>H$8N&VbiwDtK80Ps2VOGB`z0~6kLUC? zpOd#zJ*$ybyXa)JXuw1-=d{wEN{oP&STJV(!k~xFiuJJgd@7gm_R5cX47EknfpxbH zD>qmq0)XmcO@qTsfz$$Z^L&C8ahAtE&a!a~->r5?QRAEaD-Wo|e2;N$*fi99jgrn5vg`#y?^r`haq4Slofq5>$im_`1-QKOk z%&;mGB>}IhRjuwf*~^7*^9Fgtv^d4As!=rlRU|dvnu6Npc?qwdWxo^4eqvLBZ%0%D z=hO%(N_4eb0GRLD9%tl1LfDChb{UB|%m@{=mtT0%!k7kK@~Umey1Vn(w9fdRKhC0t z^Wk(4KThQ3Hw|R0>pwjgoiDaemw_XLWr<{z)_aHx4GicAWee9?!Et_tgQ006x00u^@m(?eFkc~{UMg#J`2e$h4V1Sy;L1#sRBJcST}p;zF$OUT5X z5>2|Sru~tH?MpQ_%r(ywhcrs-XGs}&e~|SE;`<52u|sfJ9*9-G?2K}I^QkB z8y80%2S-AO9SOkBn@`au^%YZRYcmdCFoafmQ_9 zw@;rwrvGwo`{Qb>?>_m-l0lHGZKm#%^2hIACZdZCFAE&n1HQ+spNrP%9vymiK--Tt zMd0Ewk(fBS6YqO&Pv=b*{9#gs*1d@O4i{(|9{ApmzO^4ckaCAgMMnlxu_smdG}sbSjVE6v2)r|B;=yMMj*?jR>KN6QCX+;Bc!y`0ATBEFGWqT)I&1V*YT8~o z9^7nh$LM3mGXc*Tb`R(41-X-OHF2zv0e&gQDL96mJ4`s!Z5npV##4NiX<$6|QR*CAhPhq}RC?J)BPnl|w%<0-s}i?cj_bzCs2{qf~?NpsCu;n3lG z{T)~+T}Z=nc#9TEHrf(A$-z_%;c{Y2g~w@~Lpn?pbx;-MrJpzgGb4v_3#Z1w*L239 z<8o$Fm)*G%&Deg5?^Pv}Oh1n^jXYZRKI7MvSrIj!^_D)r3|IUlqQ*2~U0&2<)3mC- zu!yt5(gA=J3kmxU|oaD2?ByY&!-ox4QV z?9%n2B6+N?%BONa4=_`1s^jr|1zT{Vz9`j{<-`8Yn%Q$sf~D`c=!ALFkk= ziF`Z?%-=|+B<#?D<&!hLt&15AqDCxuA57fMrQA&NvTKtzN13vCS(YVSICyV+nC^7V zjP!g)#cBTFU#a979^QT^VF$W#KW`S7dFv{`-uR7(nG?}-Zamdb80s_~k(p;X{zMKz zu5FB+P)*T3cyzkLfEy)+bd#&1GDpStdA@6em|Q#(;4ll4UiWm7Gwdj#z_8c%22$gB zJgliF1|S4J$zUEv=x>fIi_3fvvwdfZ>`f6Pe)N0tXiUW$>|o!JrR~f1Pz=6sJjyex z(%A6T4iR8cj^@2|Z#nBcaYmn(049k;Nbt}hg+ue;s+sMJVd%D6hq>0;(g?PubR-~m z+np}exb7rb@GWv1s+`XhMl4q!d7}*~BpeC&Bn2x?H{s$E0ZL0OQV}f1m6ulh(%;%9 z!L*Oz_IQjM=WCq2FhX=v!|amRUD=ET{H`EM+LZXv<}O2n?SdeEX^Ro06DTQo%q&v8 zfBI=BPWA@-4a>LP#)oD%YlQG|=}M4QjnO~ontRwY&Q}S1(9mnMgVv8fj7~^`uflnI zj+V=cJKI7A4bb7d&E`Up@V^c>5n_AHw7hi|;>>m>a?5np-Cn^Q`=UprLMEERW$jz~ z>gI9c-)IGRKJ#y@y}{hcS&XZ&ZbiB2=TrDcTh63YRm`KqG2CtZ_U-r?os_uzsoRCU zo@WJ{LUcTH)9)d_0=&U?RrxxNqsFUMp7)N6V6THz!@x7}jG=X>_3KRY2quS)R;kPG z!4Cv|lOWN}tzP~WHT*3M3MKzKY|eG(_*uxiyS6NjExM!0~!DFj_K$T zti5h)W;}cCOc%Uyn-mJEhXE{ul;8;TZ*a?F*&hwvvMdbpv2GnlW;%zNTGl*fDsnCas^Xi)AFhLJ1kT>X>z=>iYhAqh zaGal^0f9autA7mfxb!{GIu_2`Y-Se|qO_Z3xeX_u-`j8;r{gFs3@XWP*uCo}ni9xm zK4#kH)FlI>IODWkj*oL9`Dq5QW%lsJ!o$%xwSjdlC@sQK|}k=C+I zyYm?jir3|{o7Y*!5v_wv3OP=@yc4WhXOq|9s7Y)iUx5?Y61A2jc%gB|lsb@~{Yo9; zhrCaKro~JbRBrA{~S5V|rYUGLckeC++)wWX`HaEJ%&jbt^lE zps7Z)JdCp!>TeI-`+Xm@InR+h+?1|_>x>}&(;E-6hk%)SXQ=d50kp*~`LLM%e9tZd zg_EpmyYZb>tHrzUl-Cf}LbEtvT#HR%tVLM@&1r9ghcIDw5}>8M%br|Q)n%-aLu$A` zk!~m5D!zwnDf2eI;D#MMmCmy%%R``DAKxOgZwdK2CTpUbzftRg*mAOv%ku26VcO*T zGBUobV!f*`GQqJs@Z6ul?KJWF(_VV2?6>P2B;mZa>a3Yr!bq3t1ULsjSI`%!;I0IX z#QOM0ir~4&0$%kbX0#no?+tRBSl(2Nnvl9FaIX;1m)|*T&^n(Ro2>eDyePmm6OPMubt{D z>Rp+Q$pLL=Uz7D68hG!gfgEUBxvbOKzK_e{ktE;N;j>JEXnRF`H82C#KLOLosf9-A zLkuwx4$yc9u+l{Zv6a>dnem$`xLVWV-0h%98w%aJy&=yp;rXD=c~^2T&}Rx#<@4=P zJiV6dyNMi**nS?BMFiF7M?`*smF*m%KIjz!Rmp2EmBlFXZrK4q@R-PiwB_Q8 z;;J#<9F9?_nEfi_E0#AuuVbm5Ghd1k|TU0|9LV|-l9IwdTDLwmcx6+Hpm8{HX? zEkjl3ZNS`|ON1*y+h|Sb1<6pBb2OWcx`N!KU5Q#w;%FDm%A|8qS$LprHvq z=;)Ua76I2LO&<{<`9a1#ipRG(s0NzbS(ih7z-QY$N8z#SP)w>X_DPjoe&JPuhkqnY?u@N zFlJ(<-$e7daejT=XuC+y5yv6$+W9k6p+|({IPU?deV3Auk=xy*{Fud@dbwMYtL(Z% zP!ZP9^{Xi=G0Gq20OWaJtfQC=duV7%2mseU&|9~p_+lpi$9+$N8S9{!zNXx&OKX}0=|G0GOx9Ojt0)l zAQ$)c;m5F4V2nokiD=77PcBEgQ0~J~TgB;$xt)$%521H&j&X(j2F}e%qWb~vR=Xql z7;9Z@Pay0nCndHZSWsU!+3)1)@z|{xl30PLDGNjO)1{V7>-ma0e3nbevy9a-9sq5} z?J=S=P>7RNB=fRL#l>u8WCh0N*YfC9@vF=v;LiJ~Q%)n5$@W*qW43AI%PA=H*Sg!1 zK{Vi=38y0B;y~9qq<+33C%v*`!mKOhfg;jNVUp`Y74KDX5lw+leGM^`Z-pu zjip2HU_04s0Xg$xn7}!LV0x##u4)tS1P1sUc_kut-gI98#ngio`cW4$7sCu>*K;?gp`* zCVXx=Y>eD`>% z_~UrJ6`{uQvvJYNG!f^_romis?}@wxd!kuJ{{SLn zbBKR10~FZ`?mn8tfw|uHpiX6+Q%HvGHr%zdVDY!zu&@C7x*F?o-n+6il_Kxw;Im50 zjoQ?`h>0v{`Rnprk4)x6UJu7PQjWBAW}l+H^%@l~V|3QfW?-_}0&j;-eLZX4=lGp{ zJ(Y!3U1^oBea|aTIy&zOiI3c^v41U&UFo*QFYJXw5_-y_c!c;fH@Sv-~ipT#dBr zY|s;nYFrw>&L^b%eX~OSLC~~D*>tjlbGb~Joo;B{ghA?MMofj}E{to&;W3f^edRkJ zINGx8I$zLHQE(^XI9>vH&y++B6UtTel6Xe6#_q^m_MLwjW2e?6&x^25agTM72i$u% zN~ha;sTysMYZd}2p^q@8#=9CUn%9!`s7GozY!wr0O;XJeU}zdKrvp&yJ1c-PRSLG*5>@6XtRm_-E9X>J#OIDBe&n&ZzWQz zz|<90OWZ8R_AUIQmNAFX12dYT_qm6N9}3~4CSN#`=`@)ZH~h|4T+q(qI?H%I-|^K6 zf)?c#SE98~PF)fEL@w)c`0dW^052Cg2*#{7O`F5e=0s(X?V%0pa&hW5JS8|*XzcTh zDk=^4eRk&G+g_x!8;*Y4)HiBjVlt!@vnVzpogkMFv_96V%IV-Ol-ZdYC*1%aJjMKG zlcc*+w?z&fmb;GkV`3%>5*N!86Djq0f;EEbsCTc12$T4Aju;j*Vaj`lcBTYlbzfP) zGNW8R6@|*e0)>;}v>?@JcpfgyBi7+?5hALsR?2J6mdztBUI0AcaOMXI%N;Iff!-S5 zCAw9ENS1wv&La7phm+;kzK>0iD`5HO%J?RA6&atIkghjP3A63(q<=~!mbr^0^yvBS zJADw^^&5;}fmd=A_otHNw5VW>Ptw`s_#lqye@0vHwG39<2@U51@Y-OlO273eq+wR9 z91<5tHJY*~T6*@z=HDH2&fRV4sh)q4_ehz*A_D!ff9d_Ivyg(2ME^1d$N+j!3i`KH zU!&r4`B^^B6n3Tay{yORH<=V{L-Iz<-wjSolE1>KVncBe^l9h-cY-9PLr%s>k%X6d zx@K7+sQnx*gbU|!*7l`6=r&WT2W)Ype4Ps_NVk8F`=#??@Z08&UbpI0P7me@tK#*O zWZnt&UA4cn5kJDYr)-~0Hp7uTr^V{q3Q-YWhp3A(255IvXQ*T~vOVs>c10E?%{qqK z9h-YG814mYJfj#*Y9M z2oVauOpeLvR9HsKYVK-M!_bS&;%ez|A9%+I!!9WGv};47_r8Y{G7W=JKO_qNJ~u(7 zWI)FMON%N~0CT^1yL($4i6iNcao0i=zl^ilmj+WigZdh8!0n&k>$y4QnO5mhyF)Ay z5zo%`KNVIhRpD7Jgzt~6kE;ao-?f$BRR+G)J4+{RDPvOVeKmr^pqvE41BWz_9a1~k zCRcl3L#legbj?%5HFW_1bl7+i8lT=MbrQQ7v%u4uWo1l02CXVKsWW*RpqeVlan8rQ z3EEY-7NHp?kz6%40W~hJLeH`iAj=)w;uCQe*c1;>Rjm^ZjUB&Sv7Rm`ebpD!`&Q6a zqs%o7lO3~&QQ5gT(y>-3B;}4ur=zoXRnrZb0Iy|T!Ckdvpcq8l>!RycIa?1-ZCt;@0CI)`xld}aH6-xKm=@~j!uyVuk zb8T>18qLuunPsxlrt7OWVB?$w=a{YIQ}o4czBj3sRK_etCtD7bs7S^wS-!DhFR+#HcNRrLMHQY`p68#hk))h^PO9(ibX==U=;e+7@qo)$fg>=EEtSa5tATqs8Ti=u<(a z7X&~-s3%5c+R9wQ=y44noR@UTSh$F+yhpXSVD@TvKymspX1`Vg8Q~X3myD2wpwWb! zT!}TqAiO^=`g;emn3IKY#l%=0yjQLLZwWCZ6fE}6PCbw_1E_?>@e?QvOIL2`A0|$|MO~&(x2yDxhJk+=M$iw<_rh*_tBD z$f z??9J8RvqeZJeQAeK+UQVW14UD*BwLW$IIqTVeV-a-foN5Qt=R?3%;eZY+I%z(r$kt zLS-|UK_rBiL<<$IhAu2Dq|Q-@8HE~9A(ozKC~$$3{sn~?#(#45s=q*Wl>}cXMFpl)PX9$3vgjn5*mj>+KZcKtszWCHkU&*&IE!Y3Q{Ibzh!OciCjtg*jEkQ#BK^!=raFO7taFL=ebI!P)ttOMC1y- zH=^yWL4@gX*9Lly?1fUXrUk{i=MH|isb^2-NPhvnaD)DB~cKA^Ok2aM9R&oR%L0(*$0jX8H%jn^7i|Dv-@iDQqC7Fh-{@x*M z+jZi(s@?2gClMw#^rJbeqz&5mCYyI-{WawW{|-3Kp|9WI-E9ejL&+9b(9m0Su@Oi4 zE<~s0^*T!3ZKT}$JpE<0;LD|nRcr*rwyT~QsrLjEp8aT+fm9CN(|}f@L?|EKnTyV& zfv2wj1bX}YtJxQ?p^}Lq+Qd&W+n&1Gi;mOS7xvUgziz*T#ZOVYV-ldqHE|TT zYuCIZw0#Rc;pCe7g|K|w1q%OIzI((U|MgwR$MU*iABS(o1lZ^}oE7y;I-f4s$b-dp zt=Yj92)6mMm9)W*A&=)BvLV8*0y|4 z+2vr4Y}*5A4{R~@6CTI-dAc@Q*>sVd4jM*_El{Ls5{8$RGUDre(T}#>wDklThIw@ z3wps*kOOXSaS@)Y2XC$&O`U)i5$BA1=Di$8iZG=RAu3c@>z{=1z z8TYjd_p3{HZxB*W>aI8+4MnHz?JsDF_p%%E8Z@0PWvW_9+i2KCK+9x3--q2XxdDxq zU;DwYoUab%>_wuthk&zBK%*=Ud4YHVl~%4O(vX3vSjgZg{1kKa8v4p1wE8Ftf~xj-R}9CLF4JahMGV_AYF7~>5o_!=5{cNm<%7wR(icX75a zU`VwrFL5!NnxEl*l(wo|8w|<8DWmNu_Er1Q+2L!s_g_fr(xj(DO1i z$uPj`8j#^vV#3Sue2CgsG>Z8AaA9XaCp!Vc8Q5C#YV?NQC8$;=d^jl*?@isr8gB`< z*l(4VRo0>auELg8FYkW2KfYXa*|~Kg;j&Q?>h8aTmT|_q10YR7=)$ty0UxiD?nUG$ zn8}udtaMzAtg%#3Pb9GRiMR$p?u`Ucs}wCgMbsD%X*V*&0ugqbHhtqkpTP#LZZVZ* zR8!X;`npI{i5Kyd3XQ}DQ&-nthTl}z_Qy;k!D(P%e?XLAUwlmz>21gG};n>4G;tq zp#VPxzZMSvHyl3K&QH>y7;g6O(}HH>DMl2;{H!{E{8^YuA-H-b7g3(rZ2iMQs3kp9 z<*QOuX><^7SP>gG#p4m~{plbZ%NGL!w9$-AyD}U{Ix+EauP8=;)v8s=1gXWj>Gk^# zFbr<{bN@BCF9gr_a0B6>R`R^NV&PBd2`^bQMlh&fR1-K+M^U*Hk@0_SBJ=sZzd8=r zxNFgA*Hfk1_xymC^1D=}LxzG@uGx^FlG_3yCbznA`#?^!kfMQsfcq~Hv*N#?#{-^3 zR;Cs#nH2i8w0JZaT-C@9#6Si*K*z_H#Ak>T#WJeE(vW)<7+EAT#6ACPN@{dwV)XxS z-OQPy9CKxleb)W(fvM~0^$8ls7;jDZ-=x%OxNOF~3+mk`cT8Aw#d1Pe;{}JCZsBXA zOp9+e%Sz?0{9HFzq2o}3!lPWxEsH1Z|Mv9s^yza>SUDZ6bluI}JpEGrOIDE%rR#wr@s_m{zkJ(_O#vd3C+a}^NbZ9$}I+Msc1S}UGPri?$cdg>;C@o z1nxmwlVA0}huh-^cf}0*YnO`e~dt9|Fp%L!~p7dJ^;1Zc3o*t12ECAMGAwxu}{xC@bq z{aOv*l2C>0x~)#UAT9`=n#IiO6Y=rPp*70qdLQWP^05RvWfmG;;FO8@D|2FngOhvw zUw!o{ke%cUl!_H(RMwoxSD&H_-dw(bsTjC%oa;}K(?y7JT1KF>ieJG$d2Qd5E|1s9 S0nct=VDNPHb6Mw<&;$THg2_Sv literal 0 HcmV?d00001 diff --git a/examples/flask/README.md b/examples/flask/README.md index 3f6b02e3f4..12e4ea97ff 100644 --- a/examples/flask/README.md +++ b/examples/flask/README.md @@ -55,11 +55,11 @@ python client.py ~/Downloads/resnet18_pruned.onnx ``` Output: ```bash -[ INFO onnx.py: 92 - generate_random_inputs() ] Generating 1 random inputs -[ INFO onnx.py: 102 - generate_random_inputs() ] -- random input #0 of shape = [1, 3, 224, 224] -Sending 1 input tensors to http://0.0.0.0:5543/predict -Recieved response of 2 output tensors: -Round-trip time took 13.4261 milliseconds - output #0: shape (1, 1000) - output #1: shape (1, 1000) +[ INFO onnx.py: 127 - generate_random_inputs() ] -- generating random input #0 of shape = [1, 3, 224, 224] +[ INFO client.py: 152 - main() ] Sending 1 input tensors to http://0.0.0.0:5543/run +[ DEBUG client.py: 102 - _post() ] Sending POST request to http://0.0.0.0:5543/run +[ INFO client.py: 159 - main() ] Round-trip time took 13.3283 milliseconds +[ INFO client.py: 160 - main() ] Received response of 2 output tensors: +[ INFO client.py: 163 - main() ] output #0: shape (1, 1000) +[ INFO client.py: 163 - main() ] output #1: shape (1, 1000) ``` diff --git a/examples/flask/client.py b/examples/flask/client.py index 1a2a204835..70a290aed3 100644 --- a/examples/flask/client.py +++ b/examples/flask/client.py @@ -18,17 +18,18 @@ ########## Command help: -usage: client.py [-h] [-s BATCH_SIZE] [-a ADDRESS] [-p PORT] onnx_filepath +usage: client.py [-h] [-b BATCH_SIZE] [-a ADDRESS] [-p PORT] model_path -Communicate with a Flask server hosting an ONNX model with the -DeepSparse Engine as inference backend. +Communicate with a Flask server hosting an ONNX model with the DeepSparse +Engine as inference backend. positional arguments: - onnx_filepath The full filepath of the ONNX model file + model_path The full filepath of the ONNX model file or SparseZoo + stub of model optional arguments: -h, --help show this help message and exit - -s BATCH_SIZE, --batch_size BATCH_SIZE + -b BATCH_SIZE, --batch-size BATCH_SIZE The batch size to run the analysis for -a ADDRESS, --address ADDRESS The IP address of the hosted model @@ -41,11 +42,65 @@ """ import argparse +import os import time +from typing import Any, Callable, List +import numpy import requests -from deepsparse.utils import arrays_to_bytes, bytes_to_arrays, generate_random_inputs +from deepsparse.utils import ( + arrays_to_bytes, + bytes_to_arrays, + generate_random_inputs, + log_init, +) + + +_LOGGER = log_init(os.path.basename(__file__)) + + +class EngineFlaskClient: + """ + Client object for interacting with HTTP server invoked with `engine_flask_server`. + + :param address: IP address of server to query + :param port: port that the server is running on + :param preprocessing_fn: function to preprocess inputs to the run argument before + sending inputs to the model server. Defaults to the `arrays_to_bytes` function + for serializing lists of numpy arrays + :param preprocessing_fn: function to postprocess outputs from model server + inferences. Defaults to the `bytes_to_arrays` function for de-serializing + lists of numpy arrays + """ + + def __init__( + self, + address: str, + port: str, + preprocessing_fn: Callable[[Any], Any] = arrays_to_bytes, + postprocessing_fn: Callable[[Any], Any] = bytes_to_arrays, + ): + self.url = f"http://{address}:{port}" + self.preprocessing_fn = preprocessing_fn + self.postprocessing_fn = postprocessing_fn + + def run(self, inp: List[numpy.ndarray]) -> List[numpy.ndarray]: + """ + Client function for running a forward pass of the server model. + + :param inp: the list of inputs to pass to the server for inference. + The expected order is the inputs order as defined in the ONNX graph + :return: the list of outputs from the server after executing over the inputs + """ + data = self.preprocessing_fn(inp) + response = self._post("run", data=data) + return self.postprocessing_fn(response) + + def _post(self, route: str, data: Any): + route_url = f"{self.url}/{route}" + _LOGGER.debug(f"Sending POST request to {route_url}") + return requests.post(route_url, data=data).content def parse_args(): @@ -57,14 +112,14 @@ def parse_args(): ) parser.add_argument( - "onnx_filepath", + "model_path", type=str, - help="The full filepath of the ONNX model file", + help="The full filepath of the ONNX model file or SparseZoo stub of model", ) parser.add_argument( - "-s", - "--batch_size", + "-b", + "--batch-size", type=int, default=1, help="The batch size to run the analysis for", @@ -89,32 +144,23 @@ def parse_args(): def main(): args = parse_args() - onnx_filepath = args.onnx_filepath - batch_size = args.batch_size - address = args.address - port = args.port - prediction_url = f"http://{address}:{port}/predict" + engine = EngineFlaskClient(args.address, args.port) - inputs = generate_random_inputs(onnx_filepath, batch_size) + inputs = generate_random_inputs(args.model_path, args.batch_size) - print(f"Sending {len(inputs)} input tensors to {prediction_url}") + _LOGGER.info(f"Sending {len(inputs)} input tensors to {engine.url}/run") start = time.time() - # Encode inputs - data = arrays_to_bytes(inputs) - # Send data to server for inference - response = requests.post(prediction_url, data=data) - # Decode outputs - outputs = bytes_to_arrays(response.content) + outputs = engine.run(inputs) end = time.time() elapsed_time = end - start - print(f"Received response of {len(outputs)} output tensors:") - print(f"Round-trip time took {elapsed_time * 1000.0:.4f} milliseconds") + _LOGGER.info(f"Round-trip time took {elapsed_time * 1000.0:.4f} milliseconds") + _LOGGER.info(f"Received response of {len(outputs)} output tensors:") for i, out in enumerate(outputs): - print(f" output #{i}: shape {out.shape}") + _LOGGER.info(f"\toutput #{i}: shape {out.shape}") if __name__ == "__main__": diff --git a/examples/flask/server.py b/examples/flask/server.py index 88d3e9595b..a294cc36f7 100644 --- a/examples/flask/server.py +++ b/examples/flask/server.py @@ -18,21 +18,29 @@ ########## Command help: -usage: server.py [-h] [-s BATCH_SIZE] [-j NUM_CORES] [-a ADDRESS] [-p PORT] - onnx_filepath +usage: server.py [-h] [-b BATCH_SIZE] [-c NUM_CORES] [-s NUM_SOCKETS] + [--scheduler SCHEDULER] [-a ADDRESS] [-p PORT] + model_path Host an ONNX model as a server, using the DeepSparse Engine and Flask positional arguments: - onnx_filepath The full filepath of the ONNX model file + model_path The full filepath of the ONNX model file or SparseZoo + stub for the model optional arguments: -h, --help show this help message and exit - -s BATCH_SIZE, --batch_size BATCH_SIZE - The batch size to run the analysis for - -j NUM_CORES, --num_cores NUM_CORES - The number of physical cores to run the analysis on, + -b BATCH_SIZE, --batch-size BATCH_SIZE + The batch size to run the engine with + -c NUM_CORES, --num-cores NUM_CORES + The number of physical cores to run the engine on, defaults to all physical cores available on the system + -s NUM_SOCKETS, --num-sockets NUM_SOCKETS + The number of physical sockets to run the engine on, + defaults to all physical sockets available on the + system + --scheduler SCHEDULER + The kind of scheduler to run with. Defaults to multi_stream -a ADDRESS, --address ADDRESS The IP address of the hosted model -p PORT, --port PORT The port that the model is hosted on @@ -44,12 +52,72 @@ """ import argparse +import os import flask from flask_cors import CORS -from deepsparse import compile_model -from deepsparse.utils import arrays_to_bytes, bytes_to_arrays +from deepsparse import Scheduler, compile_model +from deepsparse.utils import arrays_to_bytes, bytes_to_arrays, log_init + + +_LOGGER = log_init(os.path.basename(__file__)) + + +def engine_flask_server( + model_path: str, + batch_size: int = 1, + num_cores: int = None, + num_sockets: int = None, + scheduler: Scheduler = Scheduler.multi_stream, + address: str = "0.0.0.0", + port: str = "5543", +) -> flask.Flask: + """ + + :param model_path: Either a path to the model's onnx file, a SparseZoo model stub + prefixed by 'zoo:', a SparseZoo Model object, or a SparseZoo ONNX File + object that defines the neural network + :param batch_size: The batch size of the inputs to be used with the model + :param num_cores: The number of physical cores to run the model on. + Pass None or 0 to run on the max number of cores + in one socket for the current machine, default None + :param num_sockets: The number of physical sockets to run the model on. + Pass None or 0 to run on the max number of sockets for the + current machine, default None + :param scheduler: The kind of scheduler to execute with. Defaults to multi_stream + :param address: IP address to run on. Default is 0.0.0.0 + :param port: port to run on. Default is 5543 + :return: launches a flask server on the given address and port can run the + given model on the DeepSparse engine via HTTP requests + """ + _LOGGER.info(f"Compiling model at {model_path}") + engine = compile_model(model_path, batch_size, num_cores, num_sockets, scheduler) + _LOGGER.info(engine) + + app = flask.Flask(__name__) + CORS(app) + + @app.route("/run", methods=["POST"]) + def run(): + data = flask.request.get_data() + + inputs = bytes_to_arrays(data) + _LOGGER.info(f"Received {len(inputs)} inputs from client") + + _LOGGER.info("Executing model") + outputs, elapsed_time = engine.timed_run(inputs) + + _LOGGER.info(f"Inference time took {elapsed_time * 1000.0:.4f} milliseconds") + _LOGGER.info(f"Produced {len(outputs)} output tensors") + return arrays_to_bytes(outputs) + + @app.route("/info", methods=["GET"]) + def info(): + return flask.jsonify({"model_path": model_path, "engine": repr(engine)}) + + _LOGGER.info("Starting Flask app") + app.run(host=address, port=port, debug=False, threaded=True) def parse_args(): @@ -60,28 +128,44 @@ def parse_args(): ) parser.add_argument( - "onnx_filepath", + "model_path", type=str, - help="The full filepath of the ONNX model file", + help="The full filepath of the ONNX model file or SparseZoo stub for the model", ) parser.add_argument( - "-s", - "--batch_size", + "-b", + "--batch-size", type=int, default=1, - help="The batch size to run the analysis for", + help="The batch size to run the engine with", ) parser.add_argument( - "-j", - "--num_cores", + "-c", + "--num-cores", type=int, default=0, help=( - "The number of physical cores to run the analysis on, " + "The number of physical cores to run the engine on, " "defaults to all physical cores available on the system" ), ) + parser.add_argument( + "-s", + "--num-sockets", + type=int, + default=None, + help=( + "The number of physical sockets to run the engine on, " + "defaults to all physical sockets available on the system" + ), + ) + parser.add_argument( + "--scheduler", + type=str, + default=Scheduler.multi_stream, + help="The kind of scheduler to run with. Defaults to multi_stream", + ) parser.add_argument( "-a", "--address", @@ -100,47 +184,18 @@ def parse_args(): return parser.parse_args() -def create_model_inference_app( - model_path: str, batch_size: int, num_cores: int, address: str, port: str -) -> flask.Flask: - print(f"Compiling model at {model_path}") - engine = compile_model(model_path, batch_size, num_cores) - print(engine) - - app = flask.Flask(__name__) - CORS(app) - - @app.route("/predict", methods=["POST"]) - def predict(): - data = flask.request.get_data() - - inputs = bytes_to_arrays(data) - print(f"Received {len(inputs)} inputs from client") - - print("Executing model") - outputs, elapsed_time = engine.timed_run(inputs) - - print(f"Inference time took {elapsed_time * 1000.0:.4f} milliseconds") - print(f"Produced {len(outputs)} output tensors") - return arrays_to_bytes(outputs) - - @app.route("/info", methods=["GET"]) - def info(): - return flask.jsonify({"model_path": model_path, "engine": repr(engine)}) - - print("Starting Flask app") - app.run(host=address, port=port, debug=False, threaded=True) - - def main(): args = parse_args() - onnx_filepath = args.onnx_filepath - batch_size = args.batch_size - num_cores = args.num_cores - address = args.address - port = args.port - create_model_inference_app(onnx_filepath, batch_size, num_cores, address, port) + engine_flask_server( + args.model_path, + args.batch_size, + args.num_cores, + args.num_sockets, + args.scheduler, + args.address, + args.port, + ) if __name__ == "__main__":