From bee7d68c4ac516083fe182e50826c2b49ea265c4 Mon Sep 17 00:00:00 2001 From: Bryon Tjanaka Date: Fri, 22 Nov 2024 01:55:16 -0800 Subject: [PATCH 01/55] Add visualization tool --- package/visualization/plot_pyribs/LICENSE | 21 +++++ package/visualization/plot_pyribs/README.md | 77 ++++++++++++++++++ package/visualization/plot_pyribs/__init__.py | 39 +++++++++ .../plot_pyribs/images/archive.png | Bin 0 -> 23589 bytes 4 files changed, 137 insertions(+) create mode 100644 package/visualization/plot_pyribs/LICENSE create mode 100644 package/visualization/plot_pyribs/README.md create mode 100644 package/visualization/plot_pyribs/__init__.py create mode 100644 package/visualization/plot_pyribs/images/archive.png diff --git a/package/visualization/plot_pyribs/LICENSE b/package/visualization/plot_pyribs/LICENSE new file mode 100644 index 00000000..51c2bdfd --- /dev/null +++ b/package/visualization/plot_pyribs/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Bryon Tjanaka + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/package/visualization/plot_pyribs/README.md b/package/visualization/plot_pyribs/README.md new file mode 100644 index 00000000..c00bdded --- /dev/null +++ b/package/visualization/plot_pyribs/README.md @@ -0,0 +1,77 @@ +--- +author: Bryon Tjanaka +title: Pyribs Visualization Wrappers +description: This visualizaton module provides wrappers around the visualization functions from pyribs, which is useful for plotting results from CmaMaeSampler. +tags: [visualization, quality diversity, pyribs] +optuna_versions: [4.0.0] +license: MIT License +--- + +## Class or Function Names + +- plot_grid_archive_heatmap + +## Installation + +```shell +$ pip install ribs[visualize] +``` + +## Example + +A minimal example would be the following: + +```python +import matplotlib.pyplot as plt +import optuna +import optunahub +from optuna.study import StudyDirection + +module = optunahub.load_module("samplers/cmamae") +CmaMaeSampler = module.CmaMaeSampler + +plot_pyribs = optunahub.load_module(package="visualization/plot_pyribs",) +plot_grid_archive_heatmap = plot_pyribs.plot_grid_archive_heatmap + + +def objective(trial: optuna.trial.Trial) -> tuple[float, float, float]: + """Returns an objective followed by two measures.""" + x = trial.suggest_float("x", -10, 10) + y = trial.suggest_float("y", -10, 10) + return x**2 + y**2, x, y + + +if __name__ == "__main__": + sampler = CmaMaeSampler( + param_names=["x", "y"], + archive_dims=[20, 20], + archive_ranges=[(-1, 1), (-1, 1)], + archive_learning_rate=0.1, + archive_threshold_min=-10, + n_emitters=1, + emitter_x0={ + "x": 0, + "y": 0, + }, + emitter_sigma0=0.1, + emitter_batch_size=20, + ) + study = optuna.create_study( + sampler=sampler, + directions=[ + StudyDirection.MINIMIZE, + # The remaining directions are for the measures, which do not have + # an optimization direction. However, we set MINIMIZE as a + # placeholder direction. + StudyDirection.MINIMIZE, + StudyDirection.MINIMIZE, + ], + ) + study.optimize(objective, n_trials=10000) + + fig, ax = plt.subplots(figsize=(8, 6)) + plot_grid_archive_heatmap(study, ax=ax) + plt.show() +``` + +![Example of this Plot](images/archive.png) diff --git a/package/visualization/plot_pyribs/__init__.py b/package/visualization/plot_pyribs/__init__.py new file mode 100644 index 00000000..8b049785 --- /dev/null +++ b/package/visualization/plot_pyribs/__init__.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import matplotlib.pyplot as plt +import optuna +from ribs.visualize import grid_archive_heatmap + + +if TYPE_CHECKING: + from matplotlib.axes._axes import Axes + + +def plot_grid_archive_heatmap( # type: ignore + study: optuna.Study, + ax: Axes | None = None, + **kwargs, +) -> Axes: + """Wrapper around pyribs grid_archive_heatmap. + + Args: + study: Optuna study with a sampler that uses pyribs. This function will + plot the result archive from the sampler's scheduler. + ax: Axes on which to plot the heatmap. If None, we create a new axes. + kwargs: All remaining kwargs will be passed to `grid_archive_heatmap + `_. + Returns: + The axes on which the plot was created. + """ + if ax is None: + ax = plt.gca() + + archive = study.sampler.scheduler.result_archive + grid_archive_heatmap(archive, ax=ax, **kwargs) + + return ax + + +__all__ = ["plot_grid_archive_heatmap"] diff --git a/package/visualization/plot_pyribs/images/archive.png b/package/visualization/plot_pyribs/images/archive.png new file mode 100644 index 0000000000000000000000000000000000000000..c23a5ff59d62d3778c038f2e46b940f626988069 GIT binary patch literal 23589 zcmeIaby$|$_AZQ=-~s~y1t~=o5fCY9EE-f=8bOinP8CHZ1ra0_P&%X=K?NnH;iaWp zI$mD*#sqb5+-vRMIp1~uI@kGQEy9=Qna_OY9OEAMeUJI{hKwi?Aq62G9v+dn*i~6P zyq&aoc=*cu_P}pKY#x}w7apr?%2slwcdcx-E$-k+Yg?HanOYg?-=?*>V_~UpYQoOK z!*byq?L8|iGfQ4pR^#6vU@^7OW2KYX*$x*WFcZ6FiHCPU8~wK<=N*G=ZMCyjGc`W zHr;*yEbTe&{evvV`yXY9`2nO_1um`G}*bdO2 zZ>2lEQs^8WRQ+mvEyZ$BmtiinC$DzqsE!wm4e!H_ZlZeLb-}Ibe|$_t zgh{%UAyVY9AS)&1)zH|OrB@g6W11f=liO4mcErcUeOqYNOyx9e$&AB2KPk$gUAD`) zpR(tMi%T)J(~nEe8`z#!!_*=&qq&KUI)kC=S0O^K#f#HDspF|78vW(ojq!4^Mnq^G zno;?(m8*@0=lb(g`(_(=yz4M2NdC%|p;y;p>M}=Jw6TsYwGLpAmK7I&YGG-4N_=#5 zbhV%a%ZFR85PC_!8-K^bNc|O|<NbCohN1`rrjt91_v40+1Yiwa?Sha zdmkQ<@J~og%rfl}tO@7iUM_4MSGark-R$gah?lB@!rO#|gx-E(+SlK|-&VJ7R+zQp z@1xpvBX#a{P?(UbMssts;%!k;Q6>h4atR|VHF{d+8_LRYO-)UH z*75Q2XPKB9^DKwwSB4`KQ&LjuxoDi{Zw8o?vuY&m-cP37t_RbYq*LKDI+mDflUX-6 zT$gA&of@B()+XUp(vx~p)W^@AHc3o4H_8H>Vy_ZA` z-;ck&YL!i~VOk(^%bQhPJ$fQCJopGzTDPd0Zuo-W;b?>8^6BGuhC_>36N%b3S}rsg zZrJg?DP+JHtXWzwc5jS1YZ!0TX_{x@dg8C2O}2egcXo2Un>ry&NNKh|fjhaJo8fh3 z!pX6Qx1P-|6msh7$z^4?sc-{I&wPFN8&0IBODv47P$?)W1-PN{s(hGUN{ov;qpN=N z=JW6U{iKwXiBF#FyLR;|2c~>_iThaV@Z8toIZoy2nX6w6*>*gg=a^vcj5@c}H;2LD z(TlSkS|ssn+lT#nDQu^+)w4|O{r8qJIJv`y$Hj!BFyi>Py}DW>Md34ZhZZ;1K3{F4 z@u<6>;T%pNe?WA1yeoeTBeOpf{~l^P^_dHprn z_Wq=9*WzSH;PdBnqh%}4KYAQEpii=IS0=m$;{RusRaG+Ukl^n(WMzD_Lz$oe%9qe< z?=2rYRls@s(?0LxJhDtmDe~8@J$fe{I<&luhbO$3WS@Jm2KynIzg<|+)$02w)UIH#8e5Lot=9~8X>Tk`Rv-o(m+SzS062!nAZ`>VG}e5 z-;RgdcI$JQ$jJCuSXi*wF2RGl*(6?JpoMQHpRtj@G^^NkPU|o){D@a^17!%a0=M_) z_qr4An|O_n55N7uCNB(MI(-tgO0UJa!MAv3f03e;k`s-OPrjK|%=iu;%-t8Og%ZrK ze`F^+OZ)iGtEkoL;Lp0}-99vBCoFK57Cw%5{K{_Fejg@t3CYypuIN|42a=cO&c~p& zgFuXOp1YfpbadvHr%;+)R94%<+It&M!&aM=EDBBk{Hx8krlUn;CB~hvinh^+zGyXh zCAgAckvMbeD`mZ)4ClV4c2*;$@EmMx^gFIH11E7_XRcZYn$HT0-4)|+m!p!G%+FU& z%}hB6*1$g|EUpzbFI&G;p@|SO%JTL6e&pPr&xRiS&9>R8J<8b0P0S&6*@J3xy6`T- zv%_3>u}?gIoWaZ$6pQ2QaYAO9Ig(MpjMSrv9l-zoQP-nHCX@tNKNA-iPs7SWG~S%rFF{)WkFj4gVucdAQbVox|Z1 zONWjb79H@tom9Rdc2lumvY~g3TCu-%SHk?8W@@*^N4qu`0)F}B-R?ZJX%Bg5(cpfq z53iq#VRPx>`D{*quZ`O7{VbQ!o9MMEFZ15VvwQzbs}Z@hmZV$J8buC_3=ATWM=%+B z*8_}2eW{s!D;l+4o!7pcR?`$ON68-<7&!Rp2D*#VNE)f1UZk@kyIyS*Cd)7|A|fS_ zNy(ry*PPex$LoCS@s`)P{j7zr$Hq!DxYw&+vFER^j$30Vb38JuDk~e<>-b5WCg$1; zBKN|ZriYnKv}z;#@{E+}MR<6!e&aj+Z?7+@6xbwx|NcD_vad+T=~Jh^PA}~Zn(WA7 zVqr#ttDDz(-~Rf4aTRaJ&Jy(9Ksn^73uOKm3% z35{-&A~dzYdR(#N<0U5RHXhm_ovwUq3VypZ5ufA7(p}&=N?i|7^DAF@{MXH%V&^Qw zRu=DL?C&mxefA_bEPb%MWr-XQFVk|zex*`E+d|e?{PyWG5&ZEmF^TWqi8Q{GZuC0F zo_VM0rASBQMf1z2Wx~GA4pwpA`$p?L`}K7GaD=1P_fG@@8*6iP#*y|zulzjmPBAd3 zQ0`@s_K*_x4+tPVbm&b~RFvPswQJXAvA7x&)tfipB&+5a^p!m#pNzd8DJUT$)AHtu z2ZgloP;GePD|Wp-G(mxZ@n}gM-wiJud;FrAa9&;>roc8;#Oo-XxJr>jR=F=t>R?q6 z6EkyiDOanyZTLF2WP@)x*R-eD0S0CbzlzV!R($^46(<%N9Bfx#XPKMMJHV)1Utdol z=#*O>;)3*}>)2@}3yXAk3-Jr%tyN}1 z0bH?H6ou*a|1qU8h=*Lb^Q?F=D^-8y`)9fim?N2-E}PJl{ILYTg~7o=l{BsU3^HM7 z0Ny~x9j_OqQ7Cc2ahP;otSENvInxNb30O2pJhy?%*X|-57s=e-#dL zrTp`ew)IihLfCXfzN|jlPd)}I_8sh)`PK2%@nYwO#Lh$!<+|O%G||soZ{I z=f!rt5N^{5!u|U>twwGP4h`)&{f_*|kuT#!%m=CX?QXa(f22Vd;C@05@7cD;S&5R{ zL>^#fLqh|H&4eF?<28BSud65``;0*+vi4mnbY-!ju*!Fjj(s4*r} z9DuC+ty{0keW(&ED$xHU-xvEG=JUemX#Qba(D_})2?ryv2+qZ9LI%91F7MkQ7 zmis)Cy>3)&ZHNQYg#nW&gdD5xnX7yWSWfrRvQS^rE3EWM=Z zs|(}ghsVD>KRMQ%7zBGZ^KK1=!2Gu>7s}%_OI%dm-+q*ykr7ULRK@*QGATQL&@m`g zT0+8L_T5AOrbMMwKl_|$QQzW`L92}P^vfo~PqGuQbnit2`4D(%cMMVfJ5MD+Q}?1} zbIz9At>f*ZVa?0&;Y>;8K`|!!gDu!6GU=|C^uBdEMZQlGCuFAP7}T7i`tEpI^`=`NJ6V_&nTG&#_)U0ZZjBrDk!M6a-SqGVAbVG<`D7;|=S zx!FyYke&EjkRZaFFQ_Arg!kJXSsNG;9@XwGtksWD$oAYPFWh@#I~3TBPiAuUi$a># z-tC{P-f{mI;vfK6AeV-&{2QSVQ2l{W!hhnz{~Z9R(|4ipan+A9VwqBe8h^qTlL|&< zvqQg;Neq#MH^CXMvTpmJuGX%?lXe^fMJhpD*?%0Ba6O3CXASzW{ulf|xL{yFnu3xfO_*lD?$;X2jneuJtR zCc{bBBTt{EVW|gal!?-1R&z9Z-0KEJU{*n{1Fph|R^m@7)c ze+Or{joN5^D@f8mefs2{$yR=^yU>2DyQtu{j!xjKSMk0=>urnY(bZc=(g?m})Cr=j zWw5@sf+=w=Nhw)(@TTOJiIoXAun1jVwu2JnLaq}eCAvBx+$<<-TPh0@a*yNIg!+}iPlv+V2;@$DgEL7^5Q;FtjU zHh-Z-b)j@G&8|p3i-9K)^sn4zpr=10AW+mE43`P2Im)+R>k{O3nMlE8fG&Q7kLlmO zdzX-wmR4`Px-^{%scyWrWIfAe##8gm!F~6d;%=_aH(o!*n5%AhTk&5o2M@!Z@}T+h zv17+7zkJyzNX=>RVh_Z_$o2VHmCSqfr#`C{+Px<`r~FOJ`5Ei$+$Z^x-L0PDUMcBh z1sa#jcoJuHWbpyx%3i(f4hvf8^R4MC`bzE7(UPYNX81m1ppfsjnRS**E?V+={Y;`@ z#<||6Uvl;;&yQ7>tPPEhl`)EO=cZ;wRqse;ZSG=voVtHkzxpjDCTEKo8?IL$9-ZoB z+d+x>s@d5hE+eCO^{V@`XU{ITkdu-Es?0efA}{~qF%jiEC~w7O%?esRh4I;@NJvUP zta`6gp~80rrV7@v0?hegDyrnZj~<2T)stEf7x}_|88wqeB{R3FecYLn)vv|*MLNo zNJ&XUxQs*S^($X6hVa>%q~XdQ?O#C2oYi&36Y)d^Q~8pSPP0#oH*nLE`DTxj_R=`r z5L&JGoo|q29W7ewZhXNg_u}PCL>Em;{B;Nq&*}(EqT3O2a>LnyuYhEfZr_f2cRex% z_?^2s#{?X#9y+FX0ZyVbCL|<8eAVaApW{_?OdJNDi^{928X6zZ<)E@5{_^*=bT9qv zg9rHbB~~)a!;!8j6B9R^;)yQ~_eD{E?u z%|2d+^zLH>1LL)yjmb9cNeKy|kPhXNX1`ng?Ip8nNEZtqpXz>cmcImjJh9_ymHGL+ z(!u91xs+&{mf*4xsAZIUcjoL_iR%IjFS6dhUf?e`^UlhS0_Z_NK!7bsFV!-55DgAtQw1Cv*)HKIK`^6iE^^$FG z+Ae$H1Z&$|S|50XHwM*6cN)01=hQfdsy;Z;i}All3Z9kW)BMs`2mgZ2{<5L zSzZ02b-a11t01(?1~?7`x&i6N0v(f(KAWof1R$;ckKR(coU2D6A^|G8Jk|A>DfMH- z1HhKwOpfliK9sn`IJe?7yTXE#@-8d847C!ouKE_p3@Nk?p?6*?=uxPP5P*Td zLvRp{tn}gDg|hvuyZX6ogtAwMzu9bN4)go$`CZqY1ZF;w5MgRwv5QIuu^4l; zz=W{a3&9FFw6@e+`mnjk(H4pp5sw2jicrI_^00)7Zk46?S%G7DVKveA-j9b?T0!V; zO~_kdMY$-`p{{@s$!k3(|78_}Q6VKK&slac@_!K#!6MFUHR89rILQP+TEKD9Ju|*U zNY(Irnc!yqzLl!v(#HC;<~8mB-jfg)&%0LuyjS3h7x&=yXtmPo57&kh?!qsPi8?hq zH+PEB?s_(SLSSVe0NwJ(kyHSOOoD=f0|{?)XM^2|;cZm>b^PLuTeogi*4DN_yq#}R zHDx?~S`=tmLX92cK#5VD*Ah;PGGG3o4X9ZRyExeh9Mb}9RkXubS6fR`=&n2#^G8td z(!)KYjy>3j^|fK4@vK&jMilRc8xdgmZ@K<%yd7~%obd!4|cm#ZpTz)rdRj5Q*QoM)vs{8oyyr{cYZTweq3>u zSCe_Ki70zU9(C*nUd3G9Fsa;kg3Oq+nBHEFPxW_#z&g{Hs}h-hznxqjM1QS)BV6{w z{Zx|hk&|5I26nzVpMZz)SqnRn5}(oSA%DBbBe%gIaYl}56QTN_El4#@4ZGMho%}9RZI>g%y z=KjU{$NdW2hRY651-tF0r-j$RQbBK(OR$Y31SoV;r_8ZZ9)YgkGljUtp#K~_kIp%H z;aq@C+n|;j;w7cUgfXu_jTl%z%Up``^-0Kp9DFsYYdPZRUKi^EWombN>rvgJ{Q)oT z6&?7D!QFZ#v*;@G%qP1?^uzsQ;l%TOPkqumE`{Tk&M36DO-Fm;1Pj&ACNCWd)UrMP z{Mw?g?PaShfb$v!Gph8x&--LX%m}u@Toyg4<seOT)N5=YqbNY37ZVfXx9~SqRsb6)Hu8#!Wa;RsCvW|R zcDw-V9w%LwmR8W#*7m$z9U-8qudh!hE(KG>WsHfT;_@^~{EB2B_M|+pCU|-7RutlWpb~!e^eI6~B!|B2j~*yN zTm0y*H079TLK!TnQ|_gIP;aU;FNDYP?ho@@w~mP;0E@{YzO~~&p%{0;B#=>Ax?c|$ z0bxjSL02rp7&|{2Jusk4#cLH;KkO_R2^z{3AY$EGZXf3Di~(|fivY6wvTJsgU=R?f zw8;`AT0@{l0B1BmIdqZ*Qx_ZQk^p379Eb+m?WQn*50ktTjLL|=?6U1iK6Fy#-oTgV zDtQ)f&z?QYVT|DrZvZGYpOxw??Mcr1lAc*TUmiklBj6HL6!)qdL5x6=Ax z_kIrZ?^l72pb~Ua_7z->gTgx&NCWlv=J9O@Q@Bhz;{ZefFQpGT5}@^SC&9>Q9i$mz z5{N_H#ALU`3kqiwcrA3k@ZvB=Nk17CZ45hNRDh4?9{q-5v9RO8^VGY`!p^(%QQt_t zB6At&*bjknYHCSe15QJ>(|G#yX^5b6KG3M4My~TCH|_Z&M1p`KWNAxzKMXAD*eAbl zgqu~hTJ0|W3);ozTDkqs?XXv@X9vWMdy3T|Eg?{gz#p?}HV38{suI9^x?l1maT9tf z*+#ES@&;e}E!6SPd8MbPKky|XA^BLmk;}gs1>ne;wut?~bKXY`%WByeAJ9d7 zTW7L7==cf`qt>+bq%KX%#tYkL8C?(g1BanjPcRr)zc`~Y;+=)fc;=n>9ux5K#F%V;TWS(LFd&k(@Wz( z$QPv4<6hfflyi@tYGg}&Bb<{Oa;KVe2^V#62q3H`YQ(n=ykkMEFfi4 zJj2J=v1<2?ck7L`lF5`BctaFtFy%cX4xR#o`Pn>F$3!@c#f#*8t*j)>+d+q6nc9i? zp`WUv8q1H*2q6${H9FSqGnM zJAUB-IVN;_W+wL&-SKGULZP(?F!+D|EZ`hj+@$*0=q(Ib$7&Qjuc{UGVMz`MMerBw0L_nO|1B=fI!c|SAkS2C% zx)s?@Al;0@HuTPR1)0ryINa6%f|&z+wdyd=8O`o6X@qiu#%KxFV3;muOrzf? zC)m0i#*-nBKS3CS2K=^fbg2)lSLgM?dcp{aM%UTEAVCnE2O>F)=gt{rIs?@Cv9f9{ zowWj}=&XW*!fqr)Wtq^z{oSg-Vi1#zAkHQ)RaBufFw=(ZF3t3?(uGfOpvHS zcAy*geD-W$2}{|9JRTjz>l?{Wd2FZDpo-oT1VKW*!4AfNmowr=AZBHi`S<`$3Lvmu zt>D}NH^vn}G2SKt&z~P^(C`OlgxaX5z}9pHx9-OUfer)Ja{$}FBOyX%6+FuSEj)t* zajn3lc;zmx!gbAqK`Qt)q?s{fxPW3ZQXJWikYx!sA}Z7XfaI2FSW0KrLl6 zaZ17HUrYZNM6<;WAw=U%!6~7dZB&m+TAKAA9{C{dEPVH59m^k2*`IAcUoWZvvd}U1 zyC?~8cUc91qluJ+q<_;9Qlag2dcyK1rmT}J^E$Q8UrM(OPeR0f;pg{SXzioG*zwVf zmg~M#r;zs0&||XpMT5YG&o2fMJI}*V72_0$6U?`I-*L$ZN*w!Ze1`75SsBtu1IxH3Bk^lHWhvP|mDx#8H9%i|LgESzY3j`KtT5w%(QoZ2L- z)q&RbkNG?5i>gPD!~5s6GST*F&?AvqLQHHQ3!ruzC#Oe`9{I;02igKS&`O7Y!KvYs zCr<+TbHwOj_WeywO`6wcB6ySV?aZ^JH0B1=96;T>L4f7I^c9J!emcHcW@p%+ECc=@ zP>^OO4$OcY)?-a~JLX3lhy52nPcKRO1FMV#=H68R6+~3L9Bo4g8Fe(bv?S!_Ud$E1 z?Agp)YT0n}w=prwPeC()eeHrASixp50t58{j+Qd%|05#G55$$d-BLn=*vW=mk9sPV zo;`C0Sqy=}l~+=V1(XCx)}u2+2e=&yG9Cs>@KoNoR zLI6F{Kk-=K36+hv+)Eb?KrsQbTIa_M$hh?4riC+)ML^FNM}7}9amLhO=TnfEzX!|Z zEF)vRGNUPr2{lzE z{uC6GV1RYL_$H;rg=R-bMaO-%tJwiLS2`-no(hIt?B<$IQ%udSb)g|X-)!S+t1j&h zm|b3PV!bN^V1?Jr7k7K1GO7b{6`QXa&qx2X<_Z+~R%#}EyZ2LVr?M;-f@+DtL9}Q4 z=B*}29-)83L1?qsY7>juO?Q+YKiIZqyRiU{jeEP0Wuf+%++@4wNv+p&Q?J;LG1!MT z>7hTs-!c0mH|$KW-YYwPAWn3fbLzOIa4*A8F6oY4AHx>C_Zv$7MmY_8!V|q;w0?gR zpOWs{wsW{v)!Q@y{76Cc*Rybf@(Oo~_FupDc0urn_L6Po^OF0VoKWiU-mpqZ$?kp> z8&%dkb4pyVYI4cEc741fEMvn_)wX4dDJ{EveLSj=q0(UOnTm`ezRJ&# zu*E=y*$Uo;`LCxVN06rxnECm&xjK(b=b=}6uun;y!h(bES)PGw`U4wmJ9>lQ%fCuh zgHKNm`NuRTDj5RnuTt!k2j&S1BTI0ijFxO*OCI_{d`&zZ=_HHzR|7*s(qk*FG}@re zthton?xT5OX?`P93T3w(*Z|J2_{CILO92an)TGbmu-!;$XlVSM#2Ro*J*mKrAS>l~ zYif}8B2>I)ZtpXJmx?141J@&1vB=U{E?fYku=rWPL^r*_fP|=#|IszVpaXabBJ$bg z+@z!=(rDNZc%8B8^t6^k)%6W;B}0uZdq3Q-n))}V$2<@f?)*#3{l0rQ1dQ6wqj1ok zey4_k7NT(ruq04s0`RLSKYlP*yg*F&7rEa7lGLB%zNXbwd3%OlU|`@|7WMplP~k&t zHOO{$aL6ZPR*8XetvD5nbX14)DFKP|mmmuhlN^v8A>R98oVmB)>aw!3FZ}&2>TLi_ zp@>xMjOB1zHWP1vN|$usE)Wj72W$XNAqo&k6b*;bQxL7xGJn>YSziaP_&W(|X4C<> zlIi|j6w-FZE~9(o1Y(l^7BIp6hWI!zvOvCLwA-E3U*T(59n6O8u*fR}8lX27pCUZR z>StD$H$ZXps(C~72LgU%Fi2AoOu;6FGa_*3&7?aPA3Oi!;{naDeLx%$mz`OQl``K0 z%+U>kC-laD7e;qSrpCwDgCL`D>lULp$knbt2yX#vvHz*{z?VaaF1>m4rb@98 zn|=^S)ss~m)(!pu4_sGjDA2qvdT-t@Oh}vdXq#7n=idsSzes_{ueF0n+qD*(P`R$1 z%98Ck7xl%Y8MI#i;R&8`=eW#H7n++liRZPK^akS-k<50a??cC%*89^=1`{P*R>hSj zYnH=6ro<5L)#xEE^w=esvypMZ+NM8}kK8zCr#NYrjb=}v^-EVVX0767RwVRp>LqGE za%sJPL_HtyTx?ZkA8uY3T!|6!rg#g986-80OlPR)@76^m!(1X6A9fj1Q$crs>SqH5 zx_?DK!7ZRXZ|fh)mEQ`hNn-*jt*<|=X+}uxg&X3->E$z*AWZr>14rDixOrJ`+?o%+Aoki45 z)k|hMJ^k!2rgd0=^tUtV7BT}1{&RKbC}Y<{uB z)DSpZB+M!%DVy(JBGbHv-uZ_?{Fh|nW1tBkTDa=_XHUv54<5@Qj$pj?=D}!R!TgE$ zElo`$d_5~ZUTmvhue%(c?F9QLr^Ea`qGK1XE_GV^uJ;SAUjYtPQ6X@UGA>GCGt1?M z)leRy=Ce5$Q~6DPK?f`xA-vWG8EioQTHeCSM>-+91t!l@?~4m30~orpva)y`=HtMk z;CjpXq|@dL{}uz!;py@%DCQ@+>hN71=!f#T1#$X`Ff&md1H;2r;|mFp5Ly=ZC;t>3 zxfr}v!`wA)+(2y(FxzpU=7YiAKL)I)@xbx=$L!d-=N@?YR)<~J^_t=NG=ZH3kCjW& zY$X#g^~hgN56YrAP|*;y8$py^@HTb2qMmPcQ$r&HZ9U{d9K&QK7L~qk-W+Xjoc-L8 zK6HJQRzPn_0DaL+IVFXFLni$w2&9m@0_mxQT+~>L=KbmQN4~wGdq0*lv*Luww@!?( z`%Zvjv+Z;@^1gEb5rEx!IdRJo)fsf!+k^cbGbNW%?udYJb&v!vEM7wU&R|ND+)J)d zoqv4qnC6xE@9y)U4D5OfAa@24pb++A8Jyr6ved0_sVIp9Pn2})17VM8tiptYho*!x{_yN?< zByHLH(IfpCJ)*C}rAGPnmKKs5BRjzIp$ER{6K3xk9B8Z~TUDl`zh|6DoT-!lqyqmI zXl+wsng!_XC-72RQ9|L&jrifFle`L?_33AO;Idayh75aprAoD@Hlw!@;$ zn6aH+`{@|}1Mhix;cJ%=lASu(2Hgti#y|b)|AxBAmv1Ah`JV0El%Cgff;k?!zW<=o ze(_G<`p(jV4I>1MyC8K01_#HDHpV6bA<}M)5I1r$b^aI2#wI3l8@I|O{PNEe+EJ(DgnqdO%VEykdnTQjc};Mpu&%yQ482sH0tsH z5$c3=aB61J4&^pg1nX@|!4xkvDJelNr8D1qfI0~ZyWV`@y`XrMq5-yIc`p!HKTe6b79G=W zd&wklCR3Fv0bdrHkW9MrA0>HudCAMl`a=;I+BsFwgG+H(oQTiN z8g_I?m#V&c#Ps$LeMbhTUkIpK2l1YrF@GW;NgpB5xOz*_fMDOF+|TQ@oCbCT$FIM- zn>duG1G^fU-anHei`M-1HgoK|H_Mf)x{sg=8Soa+EAkaft#F`;NB|R4oFvc`#b*k@ z;di0Tg+?aMruREJDdayP!;LJxT8y)=e@qBIwm*(nXXEPQ2HAqt>44#LC9Yz;HnmfZ zUsm#Y6K7N(fZiNn^W3dEgi{hynQ^vsq@XjfQBbbN`z=81Jj=pjV&NjH19aLcaSviD zS+GG2E@4qen;&w``ZpOtA^fw=3NR8xd{NIO07Am)mX$}A=-innPwW6l`QP~aeGH_` z&m$vQ#ldLjcz!D2x*!-iWOiSl!4x|ez{&>8`D=#$<;{ddwzLV`)ICD}oy~V#oTvv2 z+}rKo3d-8jXXRkR%DPv70$Fwet~5)s7R7)YXk=V-BmePV$4-b-@Yy8oA)<(xcAj=z z1(7(ZRl`x*S70%Cs$|3EFoX&%bsVE|p9b(|^7tpzb3>3ki!8O?pf4d33h`jS!PP^h zyk|Z>Ez>SbdL3}1iNCW|0K`@~sdY~3vfkl+=SI!8VzH8pN1_{CC6jx|b7Om; zh0mMuoMhCZW6;{U71+#A|BY#&jNXdn<9^A_Jo~HR=JAy4vAg^d>I6TozYATRW&6x{er7#r(&%!* zPUbIlJE~@>gZd0cu48I0D^;40`Gr6jbpBDg7p)NFQYnJYOjM{`w58Ji%|D5c{02)3 z9V@VWaO)zGd}h6FKpSS+PHT7{WxMR3xlJj=+?ixA^Vm#KYTBTSjxpRLNQ9K*+M13)S}KbNnLknrg~P%cgvfm&-#0V zTVRXh6UJm{E!1s=toB6gDWVx)0L{o>zoflS&7cgw-B}A?}Ks0n6S4Bji(h&#N%V(N3L}W8DNpC}Ih}sI<6Y!}d z-wAlvph6mOD>dX4FEA59O9^UDK+}$VYv3)&9|+F9@DrWQ96feDm-VCL8w1CQHZ~x@ z_O@~>;DUeYfYqP_x*KnR;_y?d7lWAiyl1)!Ipn*3;eU{xAN~kMA={&H_0a_Jdlja_-(I@9jvFKsg7-EAjPfVK6Nq zYrcz%%NGH@J%1g24a%^%KOnM&FDHFRpf54nhgu*3i1rciYW?c95O4T!eOchKU0I-8saTEA*WbeA>N%ko0aJK#2U1)c>2Pp!USja{5D8*!zJIG_X zSx=ZXfd`ev;lM$ib(Q;1p)Ck{g>C9{Aae-N)o#n2zY2=9)8ou z_z`aW+QomR*6`4dv~Az)mK_Rd-`?LufBnmA&H@#L$xl)WF;Hcj^zOedmJlFnt?dgo zr)}sa>^XEhesbre{=*=i=XRRb*8_?xs_i!}Fe6R+;gVVnCj0%UFY#+T?=^eFPb zj%hR72&A5_-E6vbwRUX6Z17m8#HDwOM)8;y`C=L*5@$~G+Cv)w{pleUhp6L6J$8=O zVg#6@YuBqT4-4gNP+Ppq4FH#Bh4OU&C!WaW<%+s?*Hi)P?y0?_A;PK`6&TmJPaJ`s z!9T=7^7Gb&OG}mHxAC{CsNc{n44#dC%~HZ`|MrJqPY58GTdG8UeSc!59gHk2Er6_m zf=T`K=_+Jab$~6>*+Ni`@Kaxpt^#aEZq(W~2RqXGwiwK~o8f;4WB;q32)L9c=?&q& zVpCB!==h7da*Fo9^XrCb$nFOfyh{fapxOS zDp?QS-At^k*TL~o3dF|+cJ{#qhb*wmkbQ2;F^vRIB&YN0{GPfl*9{!9nnF`amJ80| zSmWCMoP-1kzzDzW&HqF^p)NdNj6l`^2j_ojS7Oj%)+khl7=I5Z3}Y<*6wG_fzKohA z?Txy;Ff>~=4qTmC-KsTRewr3IRB(1{GZhOmDj2ESGmWLwPkgFSzjDm^^ zg&MT#*|cMk`zZ6yWEB1d!;k;SxT<~DOVpU7)R;;>a$sOUkZm*!IXaHVKa^q0w zaD+_^Xi!2hGPRxy{`YkluB7rzd}I@9fs}vd)TzO!$qN@Q0LqMl3SNKtN1uO8IG^p= zgZsd8kBjO^g$}Tu)s`HUCWukR;QNwzfVQL)BEwV+U~-^Fmjz^wx(UDxjba7r_hZsR zr65F9)Cj;~G4Q1~8!%f#sA+M$lBRRkrB1MR-U8H?4<#fBXVCzP#^Yln8a^HnnrLf7 zjGHxJckPIs(DSJ+?jZq9YySr=4e$VX+@od$Vt{Z0xT#*mfOC}7VWksF2jt9_kM7X@ z7m-xHtIU<=;71_D>-&%t;6% zuSuE*pUpQZd(tRiK7%_{+9F`!nX)TCf-G1))jz(ePx%xtWr;i<&*t zE!620CADqXP@+uHEz-vpT6g*)b~XqWb6o0x2SLOB$znTLlif8kBR$u4Y0Dc*E)#^> zp?7X~aU%QGAC809Zv1^IZtg4PkN@cN{x?nV1KZa9^`ycB{}Y3GSM=*!-O4t)l_s64 zQAELC0v0;S=nrq<{xH6a2|p!3Se%(q3y~RH0tfC_QG%N^g>7vGJ!EXUsM{bgJe_l0STV^fjWw3 zv-JKJLoMLo&L07%V*uCjlqrKDJ0Fx^-*RVlPH55N))KX-$9d@YXJI0(e#elyoYiF2GRpEuU( zHb#K9aR6TiD~4-3Kh^Ghl7AERsMWgLt}Xj|$}L|!haWX7S%qvT!(NOwT; zL!7Kxb$K(mWil59KI)8+kR}{?fQTjt8xV>WG&HD<*tI_3L0D(jEdY*L0GKt@nA}5q z7YHZC6t#Gvjg`99D$Vt%no#cXHy+dt>b6~P!6d6fL!ttmQn2-^+QW9<{vQC%W1}4C zMLwou1x6CCC!=#KneZcW_k*EAcDlQ0;oFr1kE-`y>gZ)FZ!7r1uJU%P`(hQlj3pg? z&wVpNDG5h8`t?NLc|BK_6a0Y^P`TDf6bvRBi zVQhlh*YYJaEXD6k^=xcjteDY$iu;ro)nf8}ZGGi3H0X^Ae@#8NWHwk*Wv#k<7uA;c zeY{jR)JCRk7$>dZFchV>$lr+NP(GZ&JV?NxeCXPw#ScD8QEM@ zDe>LAw)!w~J0D37J?7p4I%C^*ShrB7x%u1ea%1=-9@^W)`~Gl>*D436YM*4F{fQcB zH|5Th3Wc;@YN{0c*@1xT$bksHLnKjoX8&DJ-NhJbbMyBM*CUcZGzX3yIg-FHn{ytR z6t~n1(mUKJrpQ2*f5ucLMGMUqp0u@=#=MMgV?5d+NV*mRZ@v%1@%qU(;bDvBp z>Z_(lC-yKKn0nP$1~M(Q>G+NUUHBqD@AU3KUz=X~&++LPvKA8iWzu8zUZoDlxE?ZYv;j)>k4o^6J}wtA*V{D9D0H z-D7fk`W)mCvR6MWQI%0wpXgi1VxIj>KE}T(p!U%QCCO3~X-`|JYJl|y_vPdrc zy`~j(F!YEjgm?9u*IK~A8%Y^hK_>Cx;}^-Ai3`aU_f8a(^UiR8UEe#pf2M}C#I!tc ziss(2!+xG;&YgRKy81Tl-$a5+Klw0znq$Y}0k&v`mY>royr}$qB|Y78FRN`%=z@=6 zCg?ETj9J~g@$if*p%)VTHof;Ef!;%CQzxnq&i5C*U@8$E$p zAh2g!knXd@1n^f?(@UE@f9}x`-wJG}i8aKTpnnE>WiWt_!M?>J4(dcK{(U!~)RHHV z52#m!YU)BE#BEfPUb=LDY;5dBWF)AIYkrP4e}U}}G|d2ddwyzx<=%(T!{-tcq~Rcr zx(J~oCklAUE}ITx{}@n}!AdI$f*l;p;oI#n&ioS=s_j6W*|NXQ%^mvQ|3ynEWs((d z0BI)S*)zhLU^X=7{Ly!z;HsvSo$CC%UUGxxV&GN4Tv$@yn2upDugT@Z-}+zI`*I?+ zv%Yz=tzO7-{_T=g?pAxW8qX#=f%@^MJ@ogF-(Ua97kRUZ0Q{5#rwcj@36yPT!0vR$ z5K6Y6lW`y~=|^A-2FLgJl@+rEW5u2Mk$R$Xt2%)t_e{n!XTG@;kU+CzEI^#m=c3d% z;aq^4H<=QK73yGVr&;UWJJwTD0zJRhHJq&@_xDoo%R3z2#O1pLEmGj>X9W=!aJo9s zj{4a+bb18HK4;)i6x7lH=3a_fCrOQI<4cC+xPRN^4R(Q^l~n2q=O*I;&&e z-!c6&CKCeH{WJH<+zCp#jB$1)|I25NWiI0I@3sDQoZYmWKjZdiLUxdQ0{r~GMHwS8 z5@fh)PuAjlsBbLP)OlR>9jHP>Q1O&k;Ed;>(d`x3tzSXe6%15Hixi;D z7OpVd#H=UhbYD1h=+Mw;Y$O+r%kmrhiT4#! z(4T7$*L`5DGV4&%pu#r=Ol$#wFwP;6FWCUgod;@}6S_W!;Fu4h>^xxfr4P)?$P$SsP?!U}34P4&!Nb0zu<0Nb3wIPLvJ62^t-li^KaeZRu&1 zKETN~jZwSk5AHi!(qjtEB4lbxva+|qUa&IA>ZHUu>phkrFhl!wMH*p40w-wGY@ zs8cUGnSu$uQZRD24yRp=W4h^p{#W^on3)qui37s+wZ^yp_Q-K zrJS4!p&liyD%SKdqng&ljVUpJoC2cAuGXxqegRO9EjMsWdudvmn_odV$*`OGF4EyT z%-6#PCv)t(6%-L+vOd|fUURfDz130m8^+vK0X7UY0eQBA(aWq08e@!2iWXYTrVW&{ zY&yvuBd5<)U^mu#HVgrZ--gWhaJf&JPqSlk(T z@>@4}%qeaDHg5*dZ|OZZp+|OK_SfJ2tW-V2se~2lh}s7g7Wz*d9u6bc9^AEts7a^9 zrAUf0FS%yW%Ms$pw!YW&3-nWf!vmpsR^x_qRDjkSg%!4MLs*4Tt;83Ua?}K^t23r|Cl5U}mm_>Pg(}&!n!qDIIM^)V zsFnU^w8*0*&0MoL#Va+Ypx?-(maOLk(k**wexAC)1%6+__&$Mba~rOO3Qj2U~WYJ!w*U_Z-v zA4{4u_D?~iMU@qFxPot19{H)@5$3^%I08MSQwJ*?VXhoteP%(hh7sWGaunrKCjlD( zVImXhy1sI!#01!ZT>}?>b*g07KrZBfs>fsT%Q0rEr9mLG!m^FpqrhuXSZb-M0f#*N z@*F7&`C+J+f`qKlQAuF3Y=!0xD`>=BnCw{c@7G|7^|6IRSdi}-b>2Qb9kI`TO5(Vw zdn?qXsKkNRH74E5i-7DNY*2$YA2g*XSSqO34Q!0nfo;ujip}#kUS3|k{qSZGv9&sS z-8BI?BK1-M%UbYVaX>dN7_%@rW-CRP6mV38Y=BNIL}y9H#l_w2;4?1_^@k>>Cr_UA z))i}jNhqLy9DO_bM8OO5R#ot=L5hj$!A;rzM<<@N6KzqLc`KOQ8^8~0HS zd~(B12@GH3Ey*gmW_{PN`i2z+Ss@O_6>#ts7ndT$^dKB|XC|ML!(j>0CXDz9tce3D zu%?Wy4zHqvBD6w5qf4%lzFg9TD}b~^sYVpfh)y?~#^*Zjz+Rxxil zL_w0rmV&_28?uhWhI&vtd^y@!?3=?zzFrHc$r5TD&&7H&Ut57Jv%{VFDlM07_k)b^JL=4#5K zzoucXs{!NIfB>}Dax$wmJFn^oSB`6>%WC{6Y`Avl4ESP0U;Jp%Zb%_q@D{6xk zXgHZ01o#GMn3}mdAsP##@t&pH2mwQCmu2mMpddYPiQfhc(DMpxEv0IXlUc?m;!|2N z=Bxl_3ZcPZ2s+8`f~tKN2vlHp%AG(~EajSo_Ylx zTEPg=AfHkB<~fx*#Y;ugyE|GnA1?ux+yoiW)TKXvp|A#b{Jsx*k8mDB&hsWM`+jfk z2gdWCi-^ade`5~(kD*|7fN}$b9-V6NlU?~9!DCt839tMe*ogpzbW4SDG1JHAHLzs6 ziuxSa|6zH)JQ!H45FoyS;T)DdY`}B%guo$cAwY45LX*xbV2HYt)C;ofvy~8_y>UaU zczq!?G$w`~ZX^ZghE<>RzNZJK?TUt6e1t{&+1`1p<)DLEgbOO1h2bl(DVtV)H=Rwg?f?Dw|7SMV Zh16Oo>HaFqxqxO={F= Date: Sun, 1 Dec 2024 18:57:24 -0800 Subject: [PATCH 02/55] Update example --- package/visualization/plot_pyribs/README.md | 22 ++++------ package/visualization/plot_pyribs/example.py | 44 ++++++++++++++++++++ 2 files changed, 52 insertions(+), 14 deletions(-) create mode 100644 package/visualization/plot_pyribs/example.py diff --git a/package/visualization/plot_pyribs/README.md b/package/visualization/plot_pyribs/README.md index c00bdded..c101c03e 100644 --- a/package/visualization/plot_pyribs/README.md +++ b/package/visualization/plot_pyribs/README.md @@ -30,20 +30,23 @@ from optuna.study import StudyDirection module = optunahub.load_module("samplers/cmamae") CmaMaeSampler = module.CmaMaeSampler -plot_pyribs = optunahub.load_module(package="visualization/plot_pyribs",) +plot_pyribs = optunahub.load_module(package="visualization/plot_pyribs") plot_grid_archive_heatmap = plot_pyribs.plot_grid_archive_heatmap -def objective(trial: optuna.trial.Trial) -> tuple[float, float, float]: +def objective(trial: optuna.trial.Trial) -> float: """Returns an objective followed by two measures.""" x = trial.suggest_float("x", -10, 10) y = trial.suggest_float("y", -10, 10) - return x**2 + y**2, x, y + trial.set_user_attr("m0", 2 * x) + trial.set_user_attr("m1", x + y) + return x**2 + y**2 if __name__ == "__main__": sampler = CmaMaeSampler( param_names=["x", "y"], + measure_names=["m0", "m1"], archive_dims=[20, 20], archive_ranges=[(-1, 1), (-1, 1)], archive_learning_rate=0.1, @@ -56,21 +59,12 @@ if __name__ == "__main__": emitter_sigma0=0.1, emitter_batch_size=20, ) - study = optuna.create_study( - sampler=sampler, - directions=[ - StudyDirection.MINIMIZE, - # The remaining directions are for the measures, which do not have - # an optimization direction. However, we set MINIMIZE as a - # placeholder direction. - StudyDirection.MINIMIZE, - StudyDirection.MINIMIZE, - ], - ) + study = optuna.create_study(sampler=sampler) study.optimize(objective, n_trials=10000) fig, ax = plt.subplots(figsize=(8, 6)) plot_grid_archive_heatmap(study, ax=ax) + plt.savefig("archive.png") plt.show() ``` diff --git a/package/visualization/plot_pyribs/example.py b/package/visualization/plot_pyribs/example.py new file mode 100644 index 00000000..97b88743 --- /dev/null +++ b/package/visualization/plot_pyribs/example.py @@ -0,0 +1,44 @@ +import matplotlib.pyplot as plt +import optuna +import optunahub + + +module = optunahub.load_module("samplers/cmamae") +CmaMaeSampler = module.CmaMaeSampler + +plot_pyribs = optunahub.load_module(package="visualization/plot_pyribs") +plot_grid_archive_heatmap = plot_pyribs.plot_grid_archive_heatmap + + +def objective(trial: optuna.trial.Trial) -> float: + """Returns an objective followed by two measures.""" + x = trial.suggest_float("x", -10, 10) + y = trial.suggest_float("y", -10, 10) + trial.set_user_attr("m0", 2 * x) + trial.set_user_attr("m1", x + y) + return x**2 + y**2 + + +if __name__ == "__main__": + sampler = CmaMaeSampler( + param_names=["x", "y"], + measure_names=["m0", "m1"], + archive_dims=[20, 20], + archive_ranges=[(-1, 1), (-1, 1)], + archive_learning_rate=0.1, + archive_threshold_min=-10, + n_emitters=1, + emitter_x0={ + "x": 0, + "y": 0, + }, + emitter_sigma0=0.1, + emitter_batch_size=20, + ) + study = optuna.create_study(sampler=sampler) + study.optimize(objective, n_trials=10000) + + fig, ax = plt.subplots(figsize=(8, 6)) + plot_grid_archive_heatmap(study, ax=ax) + plt.savefig("archive.png") + plt.show() From 0fb885d3644bc145201147c6a7d9377beec049ef Mon Sep 17 00:00:00 2001 From: Bryon Tjanaka Date: Sun, 1 Dec 2024 19:01:30 -0800 Subject: [PATCH 03/55] Docs fixes --- package/visualization/plot_pyribs/README.md | 1 - package/visualization/plot_pyribs/__init__.py | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/package/visualization/plot_pyribs/README.md b/package/visualization/plot_pyribs/README.md index c101c03e..6825fff1 100644 --- a/package/visualization/plot_pyribs/README.md +++ b/package/visualization/plot_pyribs/README.md @@ -25,7 +25,6 @@ A minimal example would be the following: import matplotlib.pyplot as plt import optuna import optunahub -from optuna.study import StudyDirection module = optunahub.load_module("samplers/cmamae") CmaMaeSampler = module.CmaMaeSampler diff --git a/package/visualization/plot_pyribs/__init__.py b/package/visualization/plot_pyribs/__init__.py index 8b049785..94e76b72 100644 --- a/package/visualization/plot_pyribs/__init__.py +++ b/package/visualization/plot_pyribs/__init__.py @@ -18,6 +18,10 @@ def plot_grid_archive_heatmap( # type: ignore ) -> Axes: """Wrapper around pyribs grid_archive_heatmap. + Refer to the `grid_archive_heatmap + `_ + function from pyribs for information. + Args: study: Optuna study with a sampler that uses pyribs. This function will plot the result archive from the sampler's scheduler. From dd3cdca31cc0a1cf4469cf0e886190136201ae1e Mon Sep 17 00:00:00 2001 From: Bryon Tjanaka Date: Sun, 1 Dec 2024 19:15:50 -0800 Subject: [PATCH 04/55] Update example image --- .../plot_pyribs/images/archive.png | Bin 23589 -> 24334 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/package/visualization/plot_pyribs/images/archive.png b/package/visualization/plot_pyribs/images/archive.png index c23a5ff59d62d3778c038f2e46b940f626988069..96a615e00a546750a8b8ce3c0ad538d70d168e7f 100644 GIT binary patch literal 24334 zcmeIac|6s7^e!&tNCTDPM21QfAqkmPrX5k{j7gcN%(k)8Y$!sRMdmT{xHX_M&ohkz=f52}-9Bx^|AA)vL8g|N7hIWqnHrGim=-XMFTiKbLTxE8+Zewd=Wy#AW z$aV5Kv$36>wXG00x5cjya9Pbq*oSlN*knqT2F| z@Hn7YucTIWsM@V`zJRxyrEf8UAjW9my&JBL}Z8gW{z%vPm~ajVlzYnMhwM!q|VU?&TP6tLC448nE|QnVk&@TSA{(Vh7>3?A{D zwWYO`-Q2v7mNve+T7IItfcHBa6)cl{jXZe?Q&UsB&v$4|CpvQ&OmRkaj2FU%RD*RK zoV-_g;*e)Om@uceMo_~_{9S-;z_quANm?P>*w-uF*S zI_!dDZ8PxpWfV=Cowbva67;z7VY@eW9c|GWo6}L3$c~iCjS03jm)!h!CW~uwQ$tSgt>{Pf}w~pKM)HYgnA9U0&cGI5r+DtkY>Kd;LtCpbi1eSby+4h zTwsj{KiSh|DxR7Mj}ESI&VAZt(UoGi55v}G(@&mEiKV3xTej>*PXI1`U|>L>@Ug0D zU}kYI6I1%|u(6bkOkQWj)O_ClwlVz2F^dV>r!xH1kB|DT1dDc3oAJ5JKV8uq&%89} zajZhBYwhsSgPp5PvANWbBPd;!@V=S0i)Wpf&WyReEAL$E7p*W8-1K1V5QS`UPu-cs zM=Z>ll*O;dCOjz53nd3D60qeT+Bn2sow+?Ho>ebuU!|I%psJ=8lxx+;m}}LQYw_{p zM}NC7OYbucD(~h{{BgUp)XgPfCoC9GOU+G*X=$yKJw@mI=Epl?zI?fQgpF;ic&4`L zh18>Lk2QDE>37urvGi8OmvGAi-1bA^j-!#QBayR9W9eh_xU~{p?8wWD-p(_FBX{X7 z>+^5)pK@Pz5?%iGHYqW&VZ0-Yfr*KPWZ3Z5>-Xx%6FkfOEuVWXFV=aqUJurx!FG#K z^`oy<_|V&b`*bESDClkk(<$S}aL^+??u3b@d5<;6aF@w~rMk6c#euJ1y*jmBCUTZW zlhd=ErmqGD2Aa$cH*^--n>~6k0{639hLn_cB#7euy9d-Qy%UO>MOBONSNtkrDvw@3 zTwHH)anT5ig-=QZw{TL!GWX;ve@3kpYd}hKf`S1(5rYIz_#ODbzy8tt&bIr}q@=>c zgOXV9*axR137&ZtC9*qBLsE#b-h10t^!ZBSX{?3SUf+7O@r<;m*m7knw;X5QG~!h& z7ZQ6^(K}~h^X<2(<(B-+Y4I%!LxgeKDgUbz9^WWs6S8_M3n+4039{bemVEB%y>lK4 zB}()u^SgVx3~+w7#jS*I3*&R2A3Vr~z4LUJ8IHm&$DZ(;e1_mFuchDwfBV<)%!`{I%)Tlu6Zq{(ZMpqjGM{wM zh?4~G#>eh0m?&?v6vT@nt9iiL$g^qt##0loMpi;7 zx|-2oFZDnUjrcMCPR(({&DhJnk@d!Tn=wPI2l9IFap(48nN~|Tjk(R9x5%Kw$uJSh zI}^liz$2Y~nA0b7wHPn9*J^48ESHT} z!$=j#rv8L`&)ZJ7ec!G@&oatE^jLnbjy&GMcQhlQ*rD}BJM>C5 z!?5~U^lfq~Q-}tp#>VxqG5H)PwJmy!3tHyLDG$`&+)SR0#hd#4$#YiEb^4BF@w7C< zX=@2Z@fACbu1ZE1`GSIi=t{VP){2`==g=bf|CM>LV>Cz?kE*WI{-OeE-^8Yvr~Mg znl~wZZA_Hj_kiGw?cBu|+q7*jXcTpKYm3gc8$?&?rVrr1Ly$PGd#eTKv6j&z7j(Qq zbiUiRQ@`rtN6qEbmKYf-h^omcDJkc8D4E1k2C9P1Cc6vt3BjCNaaG(nZ3+qsEWSnE zex^nsxv)?(uNOOJ63Kliy^EEZN_6(|_wV0Nxy+ek8Vl{mW*gBTLi zn-FX{EbVRlWNVMxd{6WmBd?)1V4G~y7WO@l6%b&;aAP1 zD9bf)?j>=Gk!ca4E_WyTG7au2DJ!?cD>CW0FAfv}&anpsAvwDYPveb7isu46_${%V zGedQ4@X|Uhc%8moSzK{v5VAXJ?qOgM1E)|}Ix^W1BNG@Jn%rCBGS*=fsVXloUo^qX z%bVl6Xw%ZtazphzTO%j+&E*^&6jh}ng3VaL%JKe5Io{Mg&M zp~1mXNDZw~XKv3f4n>a9;uh|t<$u|>Z|~ltLPCX7VTFZ-8DmE`-z+ong^2KJ{5%G8 z>HPVRm@YsuiLYKMNl8i32|J{)jNse!)B#}`8XAUHl96tL%=3N>KMKcG*VgXF5ZJl6 z)FQ>)WW|J@Lp}>j#E-R^eSQCgPTbwKeRZTM{@$t!`jOA{>D?XsUoZ$e7&pdZIKQO) zKtk@NBqULfwrgsTaWqbs^0+Qo4-5{nb8{<>wWcbcJ9qp0%nYB~(ro=U4Z{3XTP!AE z_8qmN@$$m-wk@O~BF?W&&CKv!eXIPZ4(bX$S&)>G8M{NPd(qjsfU9)jYI%9N3GDL1 zwN(OsCUUL&<7w-q!U1*-SQ4sscG-YZ-p&OF2S;Ylsw*qUaBAfm*MD~O48DISucdd9#f7R_Lt*ofCo>9Esc&o zQ{*tAWMPrYGEy?%tE{gdEw=pa+)f4|m6av@c8eMh@$z@?jA26o4rv^2h{2EPt!bil z4dB>Dt>S*JebqKuK$R7~49OE+d1Fb*Y5pu^x6fGgmFhsko?s+2D=Pv_QE1ciSiuk$ zzgKbFe|^6j9(sDqP>C-0kuwlH2j#z^uo=X{#+K*YbOu9r+WJcS_X27*1(6Y(LWo}7 zt7S(Duaimo57van_bm?!V3(&WQS1t}EX45f@kOg<8rFvG=f{+Ez+$p*QB}`*rlG5& zlPWx!e{QOH#&>SAhj@8_@0;tR&jkhq^uK=wNJ5~+l|n66pzN0AVHDNW%Cp>gL7tnFQ&mO9$vxMsjcWzqjGwL>zJ+G`MNAC-H96DpbS0De z_3JMio0|BXzhC3jD-C0ZolNr0$1>DYR>|GnJv3N*qCGRFyj1_~-JHv-kNo`FIMR#v zd}!JX`0CF`Ro_UTv|4b#M(O!ev84`zEyH`Xvv+VyY{#PKhIBzJd3{WF5x8II*?pqRLh8o`n*F)QVshMXW#MJfry` z7R>1WDG4SDhyKdb-)u*BTi~eJBn7{TUX2;3>yjqm0-CQC_|;};?6sBnbclr+E-}sM zv%+xrp{%p&?=~VP1U!C24k&|fO!Kd!?$GakwT)&xY6Bec6=b&Fy$3<8yu+JrKC-5u zfxrI3R;Bz#|p+0XD?1S zTxk75HR{}rz{H--YW-}ij9I8;iR06pIxO_spu~@Hh*e|it*wG$5d%vn({0z1@%i0M zMmWFI>3A;hHXA0<7ZVl1u?bw?SF~c&7p-!riu!K3jEK9o-m)$j;5U`Z~cThgp=_C1oV+#K@66F!8-ix+2k4XYp54Jj)r`43hMkRx(xp}tW-mpJ@f|PCVz4!ClK^b7tm3v|5&YdBmF1drX z;mI|j0(maly1EfToLV&4u*Z*gEltJ6v7#gfrI!_MC6|&>B;h%)5j!iZ^n9NOE+_L- z^ZNaT@z6|qS&m419XWCYs)XcTw|N~OdI1IhY{MG53(rqVC99{aVX@fIN>&z@_fV}Z zbtk8nCMk+7N~V|0g)}xcp2E%=!)DapPIJ5-a(YaAy9P)9>%@NI13MiT6(s?fC>qek ztE{Yj`}Vzj^X83%Ur)A4z|!1A@^-F#2^2{!EG#s-pB$E9=j7zf=4NA)xhWyRGdiCz{9?+Ti!Uhl;}E_^fpmysj0;^H#cXviDG9j+St5;(Ag<6St-7n`Nxhw zqo62v`}Ue#Gvmrlhl$RYMiI`ed*{18+>5R6pZoI0nYhD@m4P34%gBBCl_X#h#YYZ6 z6EuGeaLJxUF0LRu`}zrIagH8np5QBa>iPfxFU=~BbsvTy4(L&Nj_kU&xop1TWA(`=|V zT;Bo`&XJL)U*42jijrT0BmjXq3iyMk&!2aEVVcOZ=+4h9Z?w8`0U@*qA>cp}{cfiUta~0ETZ&f4p_xFF{?d|Q52k+spoE>cdiNT)i_UnsE4Z9)# z`Nz7iEMfd#y?QnK`5x2VBLa{Q1L}sF?(aUVVDL5LCBQ0p#jz+69FYqehNRU52{a)r zER1IVV&0;w7=wt@iEOv&a$5a5@r6&fv0tCaWLx$K?W3VdGOCOC2(${YAr@2+pzVxm z_C1!hqS?VDrVU93Znmj5To~V~Wpxb7&8B!os#MP$D5I}myXIATV0C$>4#65&DJE0B zCD_%)AwJtddABK-zNUmrF)Ww>HpN!B%GrR(Wl4|4D!o}qHp6K}lZ~8t-8Ry;YP+0! z_%}HZN8fuI930<~Wn}j5{%*}Y%kVvW_GCj|;P;z$68TP8iJs`94)^B4E_}Xc-=^a@ zf>IV7AHj%WV7Czeb>L6nMLmVhJF_wS_wNrmW#S(v|NMmS=VKP??QygW%Nl{1aTKaQwsFIY`qG4iqwq?TqoSf}`k{p3yHZ91IbH`CI3iT(nQ~Y6&jwBnih2O> z!p&klJeQt7e-1oVJrZ7FgvYZ}eR>{CO_2(8 zj%A*ku!{r1hi!Z_;1i&F~*!1O#9o(}@fOK+K0qxx3o^0xhPJLH7^thk5w(UsJ!^7Y?7za>JKs$g#M)Mp%O zDfNvn<+R3`nrHH{a~bogZ$00!n`MULR}^wu7BVqvZ{_^43lnkO8##%9I z5n`!(E^qni6d*J}4Cmys_&B(^r@o!E;Njt+1NaWRj&mfvswn(?HLz!iQ1xU3l$uhx zhbpxs%&-K|msbr8_#DPH09J`+bN{)+6sS0TRo}T&e0(EN^z7hV;Ww^lUR+ttcn0z5 z&Ye3J6|>k9&Y)bBRSI8Ox=^$NCujF|r+{&H{tdvo#f@E!#=ghHml^H7xZ7nZTuwFtEPaDPx%2KFv$ z>6!nk`%S3058%QI7p?BSeUui!&BgyJ)2}Ug>S14a;Eiareq}~Dji*831QFus+|4{e z?)B)}6^h@$jMWc<2ZdZg>Qg6omu`W&z|&Cs78!j1C)eUDm+|S)MwaFI`ZqU#AZ=gO zj8Ci^KSx9~NaiRTSVzg8JOAt0vTU;+jlKzv@+mnBCCZ~I<(4!}l?Ht7ezw#pfIR%f z8?a2H3|2TlF)@PMyUNQut*~?10Bo9@R&mFuC(2|j{kR(MS@`%E5q2DULoH&73rOJl zrgOdiyZU1Il>$wmJT4TL#Un+EByX5h&j4=R&Q|(KWY&$j?ZU$^e&Hx0S@wet*-1=l zT$L<#`4qud;2XdAl74Bg-#EE}XPZd{HZWD%zSo^T|MpbKM1L}C#0L-X@WMt=7skW5 zpSnXX?hpf&F)>a5BC`$vj$J?6zwk5WIL9enXORfCy!Z$#dvjXO5ESWO-aWS7M!dOv z04D6ru)AI=2 zv$HgnP$xptzBl1A*I|T@VOo(b!Q!`dMLvDn{DM=6mJ3x5)&+yA8HY+bE?&GC6&o9S z@%upxNCgnWmGt%X?_J%yZ(m}5zJ~2!waIihWTY3RZUj>(OFJ|4Z+q7$Q7UZT_gLX* zsbk5Ws}tq(l-`-N%k-DrzNA#lv&<70NPOFeuCT=3xk`QiJ*o52y(Ou41$gGxYS?Qh zahEYlcG)XpFAB@%&%fq6Ig^vw;LfD;2|w4|(lHgA_@QT7_|Wi^Ogpw&3Fck7U5Hi3 zM>w=YjrB4v?(W1`L(Dz@fB@jVTciJk%CHThHJ|<)uZ)$dAkL`u$iZ|+zq?6vAe8V0XeB=zac2kfvVIS!260fSIl?*{NBHIFJuOM!{G+--u zlRBAS5!vN>*2p^AAkS<-vQo#69b>Zy4h($x@#A>}CC1w`{bM2d>b$;k5mF|bD8L{5 z(&{{}FS-dAdA43b%!CpFh^rs6yCmP1TBrY(FqGWVSRr61^{D zZmKUG6Tmw1fGwhaVY>fZI%V9JrjwMKx>MK^(yb{q z((o8uy_)SXp^13^lNH9G$YIxUj**+lX}%*Twld$>H_8JbOkaRma;PO${k3M1L+T%t2NLr0PhU4QGzB5Y{RcoyK6$DOcrURr(r@h>*_jayN3ChwQX3(v1>{>jaCuLm7}`nCpWW%+6n6ow zOFqbLfE0(t*Ji}`?f4@Hz;}j&Zo0ZVPCOew(p352GwgKB>I;r%$qTQOD(UH^zj$#7 zNquwhj1lcgMn=Z)@DeN}c%eW!-uubX@tL=u=&@_MAY}7ZHk1d-v%OGk@WZ`Eag2dn z^7-vONRzSI+1VK@K(MiMaEx8_hP8cER8;i%gAX+|E!2u404np$20jJ?)LM<@NXtKa z;lhQ~3XROGQtjbVKva1w-G&g-qjq9A^VKH)jBTwZPM>a)XJ4mRs)_{_Z4KXG4)UT+K2k3hVH!_R!NO zBl+tBvsJ{!h`_*>!5dU!3sGpB0U|I3K*KgM1XOG@lxfZBdbs;x0&o$Z9D3&Vr2MV3()SiE~&A@q1-C~VU0jpM6nCEPzu=vrJx+T~|Aay|kZOJxi`Z`a* zBW@B!jdd2uwzYx+0G1eivKl06^O0PN_c?94P=_v^e?-fzcKPyiX+OqPxXQb%aiFrY zvyY~ez0g}3RR>kKjml%;ZN}*zjB^YKkEkvrGFzYzb)Ao#H3BWU{)P&qxE>)QjkN7nMa&g1>brrW@6<-opJLQ zF*V|ab&b!%zN_pUW9Dqo9XG_3Ky8tIK+k`qrwXbHzr5Z%(-oJe&hhX7EvM-!9yLSp z$HaU?yK%{O#d9TiF}v|+crKA#!qx-mahK+H%UFc>EHLak#YlFz*i<#Pe^U)H^Jv#< zJoB=A=0z#@+d~Ve{|i#BqstJsXc=tQKq+Ce$%}#-g*QMElne&m4PVM1wWRBbmseJD zK$wbjo4c$L`cKLhr#Fewt@R9omN#FUb`0>diEk@K{>+A%dO^z#{QT21?EA1)jVV&MVK7hoy5XyhSPvraklAE8hiF$U~C3*o%kr7b|Z;CokxMrX! zYQ;8+WFgi;j`3ck;?A&-FD4-1D z?%j#{n*eYmEJ}Ml=_R2rv>^Rne~#)HC{}!tk@A27GnT%skv*&tpQ5bsITxO zB1;g{h4`JN0d9|DBa--jc!8}tjy*3yw@`|a_Gv_*Vlz4AF!+INt*^noNDP51}ROxtNBOZFK(Fqp`TGu@)CU77H z0Dt>tF8<}qktRiP3;Z+S1J$|!Hs6V`>TOdC#i^>R#{uuM<*(#NMhdCbzIr6 z+uq2@P1N0h)d4Whn2Ree)^^9?S`3!mJ&Gnm*Va_7a%pDEf*2PK#En}#AL10sIRAn) zh{lO_1LO}{9YhJW6`1$!*g>VV~Esm)_W9Z~RLHE4)jTQ(A18jkU20WVvZWzhvtg41io z*IakP*&X}b@<7!>f?9a0K13ms@bmwltj;A5&RrLtoj8M#6k0KQ%U@`P)|O}R-L`cj zU?t=+y-mgJjk`|Nyo;Cx#Y)=pm<8#@%O`pY>Gu|0^>j&m2 z9NM+@C~%5_p;p0lgDkV9n7OM0S)vr84L2Xmin?e{y{M8u>-Qj&UOY)lm$TEkMJvL( zOO-WR_{5oR+w4vt@RUph+@%8HHR3VDHx2OT3m4mf0J2%O86Cti@axUm9+8xpv%1$A zC$zFq(qeR7d$@BUl@h%!9)OX@A^xTvGFC8u;!DxKQIkX2Nr3XJv{%NRU+Eo|qqQJn zN;bk01In%4Uh7E%va#hxyU_r6k$(UD9-;1yVwRwRKIMIV%B{yY5+qy;HOO@WG)wrU zbf-k4u#yE)>hbc`yASf5R{i%xs`>6oRJLZ&M0a%T4IaHLwK;$Irn}GVm2dIoIHb{R1 z=!P~*f7z3{3>l0cA@_Rl3F=pdSp(6AnA1-4`L=kg+oJuB}~A+SSe-!}!C({0dGW?Kk+c z%~Zo-wzUnxoV3?qs1v{ssxI8M4|{rS>qK`?jHy8U zC(2%sH`suC==2y%%^Q93h)W65TCY37f)m0LITd5`IVw>*nmQVfM?~0W(FvA;cBVz$ z^t_f0k`vNxc!eUNFkKmz5%=ej#x1p^zlneyt~-pA@u7POlFrI(ym$*l-%b&$(?lKt z=!)7+(~GY96tY@_b&=_pvkY^(48f{kWT#N`p^{e5H5!=$Vuu2sKToWvke=u%Qioh5 z98>xsffQ28U; zF>nGz+qP>I-dXz ziQ9kdY=m6T$j`kcd^3ft(bq9f{KG6-asCP@5N-sqVe$8KAw#`5pz*TvfmqnCY-gM{ z`4{098RdW#F<&Mq8RDZ~Taxqfff@Gufz;gId!#ZG^EX&vTh9!%sr8d5k8_V`>KV?L zsH*n0DN z={S@t$UNM;F}`IvA3n?6_!DV%-`_QyVChJqCM9(TK>oqVy7<^wmYV0hGA%6@+?DX! zq9a$vjp+Ezp8+hgM^0f--gc(S^;GvyXSe5?cU*Nj4QKRBwk!HCdAV~H%5@>@Po9V@ z-F;Yk7+z&PJp2TNPQYkHX+~%_@H-|TATWFS*s%|T7vY#JJQNI*jj4Mc`}-52;#4Ui z4ScL@7|kq0AEc$VB&&0Xik6h~&}v)XTph!$a?Cn`yjln!v|GWqC1++{62&g#WuC3u z-#kzV@RYMpmdp4*wX!K-D;40ovL}f7Ty@hi4i07TbG4T;ieirn3SI`Kl;Poxg>U*u z$p(bGvkPb5aT0m9&dbWyBgH$^s`N5=tqMW7?DGCQF4`7@a8Y->91HVL{n!Fjew&Jw zzkFm!7^j}=2LB(!(iD$D1#AN8WVKXNflZg8lnLU}!Bz~B0mC6cB>p8yPkOAn0y&-B z!w-(cpuVZL=yUhp%n5)U@WqQSv9T@<6bj0@VxC2r5z9l@kfo)z$od+qaN{ ziu(jM&rF?odVYR>z?`bKc8ZwWQucFKS^u^%i+N{wqP{1-zkg3X4QWy5(58_DpvgD{ z1k|0Koss5BM@RQGGBO?6z0}G&aUM8SAh1O^odtuTI`gjjuO9*srAw3=w~$5`&;f1x z(AT#WEQC?tzuV}zPL~5Y+XVcn-P+3h*boq?9S4Al9%l7E3g7z)Cqf*JTyVBMr4&3k2%e z;$o4ZKLGrZBna9Kv!3HEe%cW2P=f}MK$0cO}bz|x+H zSg#nV=Ouf>H|_ZNEf2_U+LF7G2&p`<>u+pE<9=rmHh#Hk%;Xnet1Y+Z2*&2weKJos z_wgc+ai5g#ItQVmY(@kO4#EqCA1z%mLTccCdFYAfs#b4lB^V3Ybek^s6DDvtj*nPP zACK(M<7>KneaqZn zP-KtR^JE#;_#?;x#qQfq6%`d@kRHA-y23v;h6&oL=;&DJZ#hlnkOhuGeD zJMF*I(RW0}L2hz`GPpC>A{6QNCBQj{2s>J`C^GCWJMg$RK_GiW2b=a!rkt!Bg~sb# zUT_wVQH-E20!veP>;N8nxa%_c*Z*{mYN95W3m0|@$I19Ho>x#noNWKlqud|#55%AJ zckkZS=Uy#;sMf`)ou5(8GL7p&%2AV;AH;_-Lf=}cwC{mLRC05dF1$cKBw$|qvkkti zTiAfb;b4YB4qPhRC#;f}cz%Y+XAmNNn@3Fka%ZMOAo%dZd}BfP!+cY`JB(p{1vy^b zgY~kKT&P(itaCjOb-?EOU|2BwZ~V46MhO~3faeEGVl43D=lykw4w^HdlL!F{5+uTA z&lqU!L2(h78gNJ=?GH!^{rU0t@88#hH-`h01kEE1j@{9Yz0NW&^OG%e6J7QLf$FHA z3dL8Jo&RQwF(OM*Snn06>L}h)b@xAkPs@WNSuy|SlN1^E&j3ya*=+(?FDL=GhU_Uo zXT0l`u+}67iBAbY_(jYzGFL3~ZGO0JoV01o2lI3%RKCw67>}>}odgZmtFy8n1kHL= zfsM5E>-*`qq$qxC2H_$`ydTsvuHz5`u(NU*C@lQwf~eYW@Mbb}cOs;Zjm-FuMg;mo z9kST}8}A&Ps>~h^at-2vkkhv)B*g)~t^tJ!vV05ZDP5vmK-Pc)x?NO3R(9)(Bd{WI z3gPUyzVy64cP#|pp|C-z0wlb?P7OFB%Q556K=(k=HKs-(3o8ee)lkH%LqQV2l6pBy zS~*U>VYDR~{2IN3nGmFin|2qTz2*;WPAnTq3ssgb_7n3^7FTTbV3g& zZ$vHl3UXw_!hGr~tK{I23l4MO4cjL(5z_`Fhy80ZTJQ`>A(8NrFi2L22b}{G_R(Ei zhx`4+eU~NRK&9ow=>mtn+Z}Sr?{Fn+sY-zV<7r3;U2giXU3xjRa%G`Xyw!8l=BMB; z0mVlG8Tn8pR?S=I^~GIy@$p`4w{CY~p%B2IJB;L{Hh6;j4@`mW%p?Cw_%;$9jC=~*c%2ed5s$M)N^ zS=q++kE#;;F0e?(_FMAtR#$E)c;-p#aMbiZk?dBtnma9GJU9#N4pPWGPU5 zavVNMGqDpnZKpuMMW0+rlcI#@mRr!2-+I9yPbo3 zBC&fSm!Tj9J)K1Y3n4O!(bP`^q#N)*% z$w0z$INocz+-Sg~22Ga@yNe=o@CvTA5t83R*Zx6ut1!4G@hJVr32Lh zd5661`R>>2zT-phN9O}9BS+TK_VD>a} zwWQ}auLocDURqk(T#w#jXXqA!wid7w;=u->0wze~nc~kP#N%b7mZKxg7SDhBghO%R z={JN{3Ai;aATSysJ2*Gyfv#qZp8Ff`MXt z`Xg_i!pXuh&3E9I?1Ue|T+0Z_#01$I;izv1??l-$qrGlj5Eb(ElJYx)&Q~}3 zD*}2qzx%W4YF~GOEwtiwt99vtRDuO=@6ZiwhBpDNrk@tKVBV87gPz$b5+qMH635#0 z)K4pI9fCiT2|e7v92Zb0Uy+(~xgMH)CEDfirV(#bKMdm|Y8)NIg2aPS((;_B%`b=U z{y}hSZta+giEgl5j;S242)>qdU^u3Ab*#rO9~`SYRUO#k)R-RTfVVHnw?b>G-#T?g ztTZ9BukvQU!K|0)Us3LkAoG(*Wzw*Xw-t6NK;8?d&JTSSa{q>r_`9Zn>tpnbZ<$DpkJy%D zHn9l3Os9ER;KXHo{|Sn;Ec(Ffg*@KoebB-L-YvC}ha72JyVm|j@*Mjz1bzSC@@{Qo zZ=9dldXjugkX`Z?kh3MBou{Q}sx%qiA;!No<#G(PLmA^oTYx3Dx`7w-0uk_XM#UR|%U z10|8;;K4KCEJ?B~T{HxqXAI~v3*Ocrf|oKAn5@h4{%m47toY7=l;7^bw)fCI*cEoP zx7ZCFYDV3_?N?@lA4V;i3F$hklT)h-1{K~72<5@jH3ggEM1>Z3d0=y#?Sj}+yMXea zXKjj{io^y27eJ$s-SEZsRbXwQ2(dGstN@iR6^QeiFu|Evi@zEGVLn4{X9fIk5!_S~ zGC;doHXhmY7~GF9V`Iazb^f)Gf%ZV?5ktk7`@N-~Eqf=>!n8stN9J$E37D~(>+7SI zmt8^m7tJPZU0op{>l*5QM_qlx?L9?~V>JSO&EOI0J~a9798Q6O0r;hR+8)sFmUg1t z3I>=XJ?zjI_KzhB%d9I9$f$**v$Hd!pWnPa3dnbL=mN0Z%?sTuXc`5CZ!QCGL_XhR z#L7a-X|044kmNt83^PyFIL-P))#C0wqE~m?g~8ci0{pryWUT^Qg{&vc&fQ)SGaVbB zZ>TpGSZy!>TKA+X33wj~)#4Yt!+@*V@WJJLWHXg+0 zsAPT6onp7UUO-EHBqpJm4g}v0y_zv=oon0$CN(!Z1M&o^BUQB~B^qfj6{S>A(UGg2+ETddhbxHH z7!~IH3O~TFi=W19ZQpGm$!(0jQMbNJ7LxAy zOixc*vytB>>USGHENF1Zqe+6EXVZiAUz7i6U3&85(up^rnKo&tt_DPK3e5RnpFtfu zny;=s0F2TlItlj7!hs+yG`J%{JzW<;SJdFy`7|iVbhJ4U=-rE^rb(0e{gj~lXznLh zU?Rf2{4gJi{#@2*57^Yu2C;-s_ecd27Io4pfg=?40)umn->(Sr7akgpMhcDcB;;aTaG2Y z01^prU)-`l?mLI379@=tq|9edX`}#>Y?E?miD^BMkVC(gWm?4`yHq{%q-EhuQ&i(e zbx_QYMAjMQq~lta^Uf7J!pMcy9y9rim7{Y)C(}^OSH~>{u7YPFuVo14j64O22wKg? z?ujo)$5KD21_J}qYd{Im@L29=!c(n%qe2n^m#8|B*FZ)~2+irA2C0E*cs!0E0nK5sY6!gmHK{Luc@lC()o@EBhSfND z-QY9~^stlOC-c&PCYVq3L*9hLFhjrGr{$Q0|R`gTZviK zI5(%37K0H4+&>8Yr&XGouuPk(vy~>OlNm@o2(4#jVyAULK9o;0cfsXh-7mQf@VnsNKMVw3gu^TD_yuB zLV+@j_kjX+mT7I8DS8MoXyDl%1%(=sf5_!SZ<&7v$f2y0lwo3%n;Rc^O>X`^?DM@` zYne=*w2_B}nW2?$&o08)+W=3y9Tc*a3_`6}N~p#t-Z!NAd!7nwA$Y!96XcTM1~{^jeA~Z$hR^)n7M33jz9g~q!)uUjpKO5I ze4`VRhWcY-V!F@_u>{t%H~&V$!-n;!3VU{n@dFt9g6xX6#Fm!o!ng<@ z%fgX3N~W1lx2Z;-$S^hl+yKHW0j4OhLpLZ4LZZrf&SP*Fw!Q&+!9>1%xNutq zQG00{bqKz~sZ*%ppAE2+GDo0d`0r*af7ID`6e0jHZDD9W!+&d%Js%O#r;x$BlL1`m zN0{SH>mo#f3|4khJL;O(cyX)Dc{C?4nK!SLGWb3fFUMGdS<8J z<~!+etjV7*06zfSzzUVFa+@*EuRmx_6l0pbjBVzZ)B@LUC0&YR^dICEmzs>%3`*j7 zjQLi1$a{P3&k!PSdKz1TR0o8d$7) za00bneN*bt-x%0FTnf_RE9=i^Q?(0T_3(oWPr;hfnfHXXMj;+qAqg@rM(I-+2PD!DO9@} zLm1sd@&0F1NY@TVk%jE5gcWy;5^!{Y+i5%Xbam5UFvh*k1Hi3~=dYt=FJCL4wy{8v zCo{!)crvxztOaHkUA>Fw^}Bnv_fP5;j^)R^sIBc3gOjjOSO;=F@q~)y=#eAk=f}_7 z{sUwKGvpTYI=pLZLPo}kY=7{J{fi2qDaehWn@?fS$zq^i5qySPJ0v7T1@r*u-_3Aw zxCWgHFt8(##aV>$XG0$z_`5N%)goH?M?>Fv|5H$f{Frb74##Hbym`5fw8Qryz8t~v zeUFthVWN{LB){lNs3@MWpzewgXd!1Yfp(GTeTgh1U@8Eos1NE1hsGBjn-8d`+f1Xt z7Bo>Xw4*^8(4q9wq$zF$Hksf{TM(20Dq)g?uiIbu4!z+>zW%==7a#i5I*hh+ZQck; z*I%$JTIcP5fT!{(AD^mY_l}RWU(l>Ob@s<<3V5~3()ECVkzJhm$c{O;ywFN&YeS<6e z8=!V^Y&1Zc+#{|+S?F6qsxx>Zw6Xb6J8@mg1d54;9+=xd-&7{O)~wyWMuC2|G+ zRs&x)TKyi3dp6~@U(YD9F?J`|?qHxQM znYsBXM9+1B&%hKw>veDs(3!G*5CK4;KT*13E7%2(xB$lHT+r3kZGVOe(>m}Pgg|qC z*)=1hP?#9dSG@eaD$c5}6m9n0drkWBLX%YD%k|pOb~*^n8Um1U`N3W(Fcj%g`kdcF zQ=_PM4r|r%s{CC=MF@zG&wwLp|2i=69Na0&y1Kol1R&t5py4i*oXNG`_4^05L1-8{ z3D|fLb=u~PyeF)f3%A@@191Y49ORqAt(5}zg_Pct4<0-iger4KPQmB9^zxRLmhEOR z&ggUVW1+7ql`B9|><2d%e;l}3Gx*`1h=D%R{!Dyn_%JmCg9W3@c;>~>+EpwLZfS6B z1-Ayn0S5OU{vbja&zi_x>swpV3wZqa3VQxxi(jeXgmmb8Ao9?e+9rw1L?s{IB)AnrsOEAClP*4LN9mg~o3Yo^@ig<*1Gr0e$wPp*NE*>f>ee0AEdqJR1skoYpo z-HNKQwN@~dZ4ix#doySMmG*6iO<{j2yn!0rauqInaT)H;95kOdtB)63(%rASibWCm^m(XpYM|M054O9x`5HJ$Yk;lLv{-`<#`p5! zLVV_W&@Q()qUd4jJt-+1>So$QMx(NmvAj| zre^@Zn&&|fKpz5>#Lzi(x2@>1jF=fA^mNv*Zgyy^9iR5CmIo6Z zSu&7%z{5{DAxCr`$AMRe1p`PPiSaP~>a4}DK@evv>lUq~zAS?Q1;tFTUS~T^25DJd zf|S7rL-H)zGtR^Xbhi%5SCp7Ev7*sb$SDLv8epv9$eR(PB}K+qQN=X@DPG{CvnJ* z0{Y;;HHJynt;mN><|nT==i{NR9{M7#x4UE)75Ac!N@&Ax0pEAVO86pPOqI>WO6CiC zFg=qB!w6BbV)Q-C`Nb{W(|)i~b@Rgw=O710^BXC%)LJ1z7aaTGv)q;I(edhfA*@g| z4n$qgO%*yevS28HJye#`0Xc5j6z2!>99`AHL*=3r;1A(L_8mxmlG z0DXpshwuA-whODxzqgY<4z{>43{ZgB(FnpPw3Ti_E(+ATSX4e%VmqYZHrpVhiF0>r zUxlp~nUvex1ugWIz6?!Z^>eG3qK`^GxJmHb-yKw7hum?m_WgbahWY104&%?DQ6L3a z#iDZCzFq;t>OWE2IVk69Mx`j6gI!R$R zS$#hRHLEf7g%ariFk3~I^V-%T5)n@A!z39nDKiKF(fp{%RX7cNZeZt+?zb2o&aazYj&NTxX=!amog8cyp?|k^&BsnC z-c##3oA@Hr=V* zS=cgAPj5+5Q8WK_s z{fFTWY|(3P#h1^um%^K*38?RKK}lZ7(!l$2fz3;B(88#isB>=PU3qm*B8)RYa(wo# z6Hrnx>h5c8Z#RZ{c!riRGr>T<&Ar4&aN4DI3?Bj$*CMn1Y8S8q%H(X(de*PW^a+s0 z3gPf|U;C))TONC!Mo!5SkiFcEF&)HA3fzyrToolff3*k<&Wo<368YwdIe-2n(!r`B z8(v8({{B%d_%j0-T#p?;ehNBW*9@RnU|At=hn36FJalS@z@(1nFi0n(UyLx$3z}Uu z6C_F+o=A^X=b6yHJ`;Gwsp;v>VC|1&wB;_^PQe(oL2&#P96NgS1Q_;s(6}MwoUI19 z(Ra*$)DR%ZOS#|>h`{BpVEBzoNO&(FDc19@y!<(UjnmC>9HqTANYJ*0MV;F|b+?TG z$Bgc)^RGtus@G%9ZKj-759(LsVp#m3=)T9auXx84xOMa)S5*UHprvy+uO#wteGht% zFoyMnzot1TT+#_Xr3GzgvaXykM!oIt6q906FX$xT36}T+n<68v4^{Y0}|+RQx#uW??$aAkcyqyQ_AlJ8DH@YWj--$5q4j z?fmi#D!Necxx;$=l+o8;(@78{?eVF6rtr%07%GIg{C?_wMDmZMk|=9~4EXV_~UpYQoOK z!*byq?L8|iGfQ4pR^#6vU@^7OW2KYX*$x*WFcZ6FiHCPU8~wK<=N*G=ZMCyjGc`W zHr;*yEbTe&{evvV`yXY9`2nO_1um`G}*bdO2 zZ>2lEQs^8WRQ+mvEyZ$BmtiinC$DzqsE!wm4e!H_ZlZeLb-}Ibe|$_t zgh{%UAyVY9AS)&1)zH|OrB@g6W11f=liO4mcErcUeOqYNOyx9e$&AB2KPk$gUAD`) zpR(tMi%T)J(~nEe8`z#!!_*=&qq&KUI)kC=S0O^K#f#HDspF|78vW(ojq!4^Mnq^G zno;?(m8*@0=lb(g`(_(=yz4M2NdC%|p;y;p>M}=Jw6TsYwGLpAmK7I&YGG-4N_=#5 zbhV%a%ZFR85PC_!8-K^bNc|O|<NbCohN1`rrjt91_v40+1Yiwa?Sha zdmkQ<@J~og%rfl}tO@7iUM_4MSGark-R$gah?lB@!rO#|gx-E(+SlK|-&VJ7R+zQp z@1xpvBX#a{P?(UbMssts;%!k;Q6>h4atR|VHF{d+8_LRYO-)UH z*75Q2XPKB9^DKwwSB4`KQ&LjuxoDi{Zw8o?vuY&m-cP37t_RbYq*LKDI+mDflUX-6 zT$gA&of@B()+XUp(vx~p)W^@AHc3o4H_8H>Vy_ZA` z-;ck&YL!i~VOk(^%bQhPJ$fQCJopGzTDPd0Zuo-W;b?>8^6BGuhC_>36N%b3S}rsg zZrJg?DP+JHtXWzwc5jS1YZ!0TX_{x@dg8C2O}2egcXo2Un>ry&NNKh|fjhaJo8fh3 z!pX6Qx1P-|6msh7$z^4?sc-{I&wPFN8&0IBODv47P$?)W1-PN{s(hGUN{ov;qpN=N z=JW6U{iKwXiBF#FyLR;|2c~>_iThaV@Z8toIZoy2nX6w6*>*gg=a^vcj5@c}H;2LD z(TlSkS|ssn+lT#nDQu^+)w4|O{r8qJIJv`y$Hj!BFyi>Py}DW>Md34ZhZZ;1K3{F4 z@u<6>;T%pNe?WA1yeoeTBeOpf{~l^P^_dHprn z_Wq=9*WzSH;PdBnqh%}4KYAQEpii=IS0=m$;{RusRaG+Ukl^n(WMzD_Lz$oe%9qe< z?=2rYRls@s(?0LxJhDtmDe~8@J$fe{I<&luhbO$3WS@Jm2KynIzg<|+)$02w)UIH#8e5Lot=9~8X>Tk`Rv-o(m+SzS062!nAZ`>VG}e5 z-;RgdcI$JQ$jJCuSXi*wF2RGl*(6?JpoMQHpRtj@G^^NkPU|o){D@a^17!%a0=M_) z_qr4An|O_n55N7uCNB(MI(-tgO0UJa!MAv3f03e;k`s-OPrjK|%=iu;%-t8Og%ZrK ze`F^+OZ)iGtEkoL;Lp0}-99vBCoFK57Cw%5{K{_Fejg@t3CYypuIN|42a=cO&c~p& zgFuXOp1YfpbadvHr%;+)R94%<+It&M!&aM=EDBBk{Hx8krlUn;CB~hvinh^+zGyXh zCAgAckvMbeD`mZ)4ClV4c2*;$@EmMx^gFIH11E7_XRcZYn$HT0-4)|+m!p!G%+FU& z%}hB6*1$g|EUpzbFI&G;p@|SO%JTL6e&pPr&xRiS&9>R8J<8b0P0S&6*@J3xy6`T- zv%_3>u}?gIoWaZ$6pQ2QaYAO9Ig(MpjMSrv9l-zoQP-nHCX@tNKNA-iPs7SWG~S%rFF{)WkFj4gVucdAQbVox|Z1 zONWjb79H@tom9Rdc2lumvY~g3TCu-%SHk?8W@@*^N4qu`0)F}B-R?ZJX%Bg5(cpfq z53iq#VRPx>`D{*quZ`O7{VbQ!o9MMEFZ15VvwQzbs}Z@hmZV$J8buC_3=ATWM=%+B z*8_}2eW{s!D;l+4o!7pcR?`$ON68-<7&!Rp2D*#VNE)f1UZk@kyIyS*Cd)7|A|fS_ zNy(ry*PPex$LoCS@s`)P{j7zr$Hq!DxYw&+vFER^j$30Vb38JuDk~e<>-b5WCg$1; zBKN|ZriYnKv}z;#@{E+}MR<6!e&aj+Z?7+@6xbwx|NcD_vad+T=~Jh^PA}~Zn(WA7 zVqr#ttDDz(-~Rf4aTRaJ&Jy(9Ksn^73uOKm3% z35{-&A~dzYdR(#N<0U5RHXhm_ovwUq3VypZ5ufA7(p}&=N?i|7^DAF@{MXH%V&^Qw zRu=DL?C&mxefA_bEPb%MWr-XQFVk|zex*`E+d|e?{PyWG5&ZEmF^TWqi8Q{GZuC0F zo_VM0rASBQMf1z2Wx~GA4pwpA`$p?L`}K7GaD=1P_fG@@8*6iP#*y|zulzjmPBAd3 zQ0`@s_K*_x4+tPVbm&b~RFvPswQJXAvA7x&)tfipB&+5a^p!m#pNzd8DJUT$)AHtu z2ZgloP;GePD|Wp-G(mxZ@n}gM-wiJud;FrAa9&;>roc8;#Oo-XxJr>jR=F=t>R?q6 z6EkyiDOanyZTLF2WP@)x*R-eD0S0CbzlzV!R($^46(<%N9Bfx#XPKMMJHV)1Utdol z=#*O>;)3*}>)2@}3yXAk3-Jr%tyN}1 z0bH?H6ou*a|1qU8h=*Lb^Q?F=D^-8y`)9fim?N2-E}PJl{ILYTg~7o=l{BsU3^HM7 z0Ny~x9j_OqQ7Cc2ahP;otSENvInxNb30O2pJhy?%*X|-57s=e-#dL zrTp`ew)IihLfCXfzN|jlPd)}I_8sh)`PK2%@nYwO#Lh$!<+|O%G||soZ{I z=f!rt5N^{5!u|U>twwGP4h`)&{f_*|kuT#!%m=CX?QXa(f22Vd;C@05@7cD;S&5R{ zL>^#fLqh|H&4eF?<28BSud65``;0*+vi4mnbY-!ju*!Fjj(s4*r} z9DuC+ty{0keW(&ED$xHU-xvEG=JUemX#Qba(D_})2?ryv2+qZ9LI%91F7MkQ7 zmis)Cy>3)&ZHNQYg#nW&gdD5xnX7yWSWfrRvQS^rE3EWM=Z zs|(}ghsVD>KRMQ%7zBGZ^KK1=!2Gu>7s}%_OI%dm-+q*ykr7ULRK@*QGATQL&@m`g zT0+8L_T5AOrbMMwKl_|$QQzW`L92}P^vfo~PqGuQbnit2`4D(%cMMVfJ5MD+Q}?1} zbIz9At>f*ZVa?0&;Y>;8K`|!!gDu!6GU=|C^uBdEMZQlGCuFAP7}T7i`tEpI^`=`NJ6V_&nTG&#_)U0ZZjBrDk!M6a-SqGVAbVG<`D7;|=S zx!FyYke&EjkRZaFFQ_Arg!kJXSsNG;9@XwGtksWD$oAYPFWh@#I~3TBPiAuUi$a># z-tC{P-f{mI;vfK6AeV-&{2QSVQ2l{W!hhnz{~Z9R(|4ipan+A9VwqBe8h^qTlL|&< zvqQg;Neq#MH^CXMvTpmJuGX%?lXe^fMJhpD*?%0Ba6O3CXASzW{ulf|xL{yFnu3xfO_*lD?$;X2jneuJtR zCc{bBBTt{EVW|gal!?-1R&z9Z-0KEJU{*n{1Fph|R^m@7)c ze+Or{joN5^D@f8mefs2{$yR=^yU>2DyQtu{j!xjKSMk0=>urnY(bZc=(g?m})Cr=j zWw5@sf+=w=Nhw)(@TTOJiIoXAun1jVwu2JnLaq}eCAvBx+$<<-TPh0@a*yNIg!+}iPlv+V2;@$DgEL7^5Q;FtjU zHh-Z-b)j@G&8|p3i-9K)^sn4zpr=10AW+mE43`P2Im)+R>k{O3nMlE8fG&Q7kLlmO zdzX-wmR4`Px-^{%scyWrWIfAe##8gm!F~6d;%=_aH(o!*n5%AhTk&5o2M@!Z@}T+h zv17+7zkJyzNX=>RVh_Z_$o2VHmCSqfr#`C{+Px<`r~FOJ`5Ei$+$Z^x-L0PDUMcBh z1sa#jcoJuHWbpyx%3i(f4hvf8^R4MC`bzE7(UPYNX81m1ppfsjnRS**E?V+={Y;`@ z#<||6Uvl;;&yQ7>tPPEhl`)EO=cZ;wRqse;ZSG=voVtHkzxpjDCTEKo8?IL$9-ZoB z+d+x>s@d5hE+eCO^{V@`XU{ITkdu-Es?0efA}{~qF%jiEC~w7O%?esRh4I;@NJvUP zta`6gp~80rrV7@v0?hegDyrnZj~<2T)stEf7x}_|88wqeB{R3FecYLn)vv|*MLNo zNJ&XUxQs*S^($X6hVa>%q~XdQ?O#C2oYi&36Y)d^Q~8pSPP0#oH*nLE`DTxj_R=`r z5L&JGoo|q29W7ewZhXNg_u}PCL>Em;{B;Nq&*}(EqT3O2a>LnyuYhEfZr_f2cRex% z_?^2s#{?X#9y+FX0ZyVbCL|<8eAVaApW{_?OdJNDi^{928X6zZ<)E@5{_^*=bT9qv zg9rHbB~~)a!;!8j6B9R^;)yQ~_eD{E?u z%|2d+^zLH>1LL)yjmb9cNeKy|kPhXNX1`ng?Ip8nNEZtqpXz>cmcImjJh9_ymHGL+ z(!u91xs+&{mf*4xsAZIUcjoL_iR%IjFS6dhUf?e`^UlhS0_Z_NK!7bsFV!-55DgAtQw1Cv*)HKIK`^6iE^^$FG z+Ae$H1Z&$|S|50XHwM*6cN)01=hQfdsy;Z;i}All3Z9kW)BMs`2mgZ2{<5L zSzZ02b-a11t01(?1~?7`x&i6N0v(f(KAWof1R$;ckKR(coU2D6A^|G8Jk|A>DfMH- z1HhKwOpfliK9sn`IJe?7yTXE#@-8d847C!ouKE_p3@Nk?p?6*?=uxPP5P*Td zLvRp{tn}gDg|hvuyZX6ogtAwMzu9bN4)go$`CZqY1ZF;w5MgRwv5QIuu^4l; zz=W{a3&9FFw6@e+`mnjk(H4pp5sw2jicrI_^00)7Zk46?S%G7DVKveA-j9b?T0!V; zO~_kdMY$-`p{{@s$!k3(|78_}Q6VKK&slac@_!K#!6MFUHR89rILQP+TEKD9Ju|*U zNY(Irnc!yqzLl!v(#HC;<~8mB-jfg)&%0LuyjS3h7x&=yXtmPo57&kh?!qsPi8?hq zH+PEB?s_(SLSSVe0NwJ(kyHSOOoD=f0|{?)XM^2|;cZm>b^PLuTeogi*4DN_yq#}R zHDx?~S`=tmLX92cK#5VD*Ah;PGGG3o4X9ZRyExeh9Mb}9RkXubS6fR`=&n2#^G8td z(!)KYjy>3j^|fK4@vK&jMilRc8xdgmZ@K<%yd7~%obd!4|cm#ZpTz)rdRj5Q*QoM)vs{8oyyr{cYZTweq3>u zSCe_Ki70zU9(C*nUd3G9Fsa;kg3Oq+nBHEFPxW_#z&g{Hs}h-hznxqjM1QS)BV6{w z{Zx|hk&|5I26nzVpMZz)SqnRn5}(oSA%DBbBe%gIaYl}56QTN_El4#@4ZGMho%}9RZI>g%y z=KjU{$NdW2hRY651-tF0r-j$RQbBK(OR$Y31SoV;r_8ZZ9)YgkGljUtp#K~_kIp%H z;aq@C+n|;j;w7cUgfXu_jTl%z%Up``^-0Kp9DFsYYdPZRUKi^EWombN>rvgJ{Q)oT z6&?7D!QFZ#v*;@G%qP1?^uzsQ;l%TOPkqumE`{Tk&M36DO-Fm;1Pj&ACNCWd)UrMP z{Mw?g?PaShfb$v!Gph8x&--LX%m}u@Toyg4<seOT)N5=YqbNY37ZVfXx9~SqRsb6)Hu8#!Wa;RsCvW|R zcDw-V9w%LwmR8W#*7m$z9U-8qudh!hE(KG>WsHfT;_@^~{EB2B_M|+pCU|-7RutlWpb~!e^eI6~B!|B2j~*yN zTm0y*H079TLK!TnQ|_gIP;aU;FNDYP?ho@@w~mP;0E@{YzO~~&p%{0;B#=>Ax?c|$ z0bxjSL02rp7&|{2Jusk4#cLH;KkO_R2^z{3AY$EGZXf3Di~(|fivY6wvTJsgU=R?f zw8;`AT0@{l0B1BmIdqZ*Qx_ZQk^p379Eb+m?WQn*50ktTjLL|=?6U1iK6Fy#-oTgV zDtQ)f&z?QYVT|DrZvZGYpOxw??Mcr1lAc*TUmiklBj6HL6!)qdL5x6=Ax z_kIrZ?^l72pb~Ua_7z->gTgx&NCWlv=J9O@Q@Bhz;{ZefFQpGT5}@^SC&9>Q9i$mz z5{N_H#ALU`3kqiwcrA3k@ZvB=Nk17CZ45hNRDh4?9{q-5v9RO8^VGY`!p^(%QQt_t zB6At&*bjknYHCSe15QJ>(|G#yX^5b6KG3M4My~TCH|_Z&M1p`KWNAxzKMXAD*eAbl zgqu~hTJ0|W3);ozTDkqs?XXv@X9vWMdy3T|Eg?{gz#p?}HV38{suI9^x?l1maT9tf z*+#ES@&;e}E!6SPd8MbPKky|XA^BLmk;}gs1>ne;wut?~bKXY`%WByeAJ9d7 zTW7L7==cf`qt>+bq%KX%#tYkL8C?(g1BanjPcRr)zc`~Y;+=)fc;=n>9ux5K#F%V;TWS(LFd&k(@Wz( z$QPv4<6hfflyi@tYGg}&Bb<{Oa;KVe2^V#62q3H`YQ(n=ykkMEFfi4 zJj2J=v1<2?ck7L`lF5`BctaFtFy%cX4xR#o`Pn>F$3!@c#f#*8t*j)>+d+q6nc9i? zp`WUv8q1H*2q6${H9FSqGnM zJAUB-IVN;_W+wL&-SKGULZP(?F!+D|EZ`hj+@$*0=q(Ib$7&Qjuc{UGVMz`MMerBw0L_nO|1B=fI!c|SAkS2C% zx)s?@Al;0@HuTPR1)0ryINa6%f|&z+wdyd=8O`o6X@qiu#%KxFV3;muOrzf? zC)m0i#*-nBKS3CS2K=^fbg2)lSLgM?dcp{aM%UTEAVCnE2O>F)=gt{rIs?@Cv9f9{ zowWj}=&XW*!fqr)Wtq^z{oSg-Vi1#zAkHQ)RaBufFw=(ZF3t3?(uGfOpvHS zcAy*geD-W$2}{|9JRTjz>l?{Wd2FZDpo-oT1VKW*!4AfNmowr=AZBHi`S<`$3Lvmu zt>D}NH^vn}G2SKt&z~P^(C`OlgxaX5z}9pHx9-OUfer)Ja{$}FBOyX%6+FuSEj)t* zajn3lc;zmx!gbAqK`Qt)q?s{fxPW3ZQXJWikYx!sA}Z7XfaI2FSW0KrLl6 zaZ17HUrYZNM6<;WAw=U%!6~7dZB&m+TAKAA9{C{dEPVH59m^k2*`IAcUoWZvvd}U1 zyC?~8cUc91qluJ+q<_;9Qlag2dcyK1rmT}J^E$Q8UrM(OPeR0f;pg{SXzioG*zwVf zmg~M#r;zs0&||XpMT5YG&o2fMJI}*V72_0$6U?`I-*L$ZN*w!Ze1`75SsBtu1IxH3Bk^lHWhvP|mDx#8H9%i|LgESzY3j`KtT5w%(QoZ2L- z)q&RbkNG?5i>gPD!~5s6GST*F&?AvqLQHHQ3!ruzC#Oe`9{I;02igKS&`O7Y!KvYs zCr<+TbHwOj_WeywO`6wcB6ySV?aZ^JH0B1=96;T>L4f7I^c9J!emcHcW@p%+ECc=@ zP>^OO4$OcY)?-a~JLX3lhy52nPcKRO1FMV#=H68R6+~3L9Bo4g8Fe(bv?S!_Ud$E1 z?Agp)YT0n}w=prwPeC()eeHrASixp50t58{j+Qd%|05#G55$$d-BLn=*vW=mk9sPV zo;`C0Sqy=}l~+=V1(XCx)}u2+2e=&yG9Cs>@KoNoR zLI6F{Kk-=K36+hv+)Eb?KrsQbTIa_M$hh?4riC+)ML^FNM}7}9amLhO=TnfEzX!|Z zEF)vRGNUPr2{lzE z{uC6GV1RYL_$H;rg=R-bMaO-%tJwiLS2`-no(hIt?B<$IQ%udSb)g|X-)!S+t1j&h zm|b3PV!bN^V1?Jr7k7K1GO7b{6`QXa&qx2X<_Z+~R%#}EyZ2LVr?M;-f@+DtL9}Q4 z=B*}29-)83L1?qsY7>juO?Q+YKiIZqyRiU{jeEP0Wuf+%++@4wNv+p&Q?J;LG1!MT z>7hTs-!c0mH|$KW-YYwPAWn3fbLzOIa4*A8F6oY4AHx>C_Zv$7MmY_8!V|q;w0?gR zpOWs{wsW{v)!Q@y{76Cc*Rybf@(Oo~_FupDc0urn_L6Po^OF0VoKWiU-mpqZ$?kp> z8&%dkb4pyVYI4cEc741fEMvn_)wX4dDJ{EveLSj=q0(UOnTm`ezRJ&# zu*E=y*$Uo;`LCxVN06rxnECm&xjK(b=b=}6uun;y!h(bES)PGw`U4wmJ9>lQ%fCuh zgHKNm`NuRTDj5RnuTt!k2j&S1BTI0ijFxO*OCI_{d`&zZ=_HHzR|7*s(qk*FG}@re zthton?xT5OX?`P93T3w(*Z|J2_{CILO92an)TGbmu-!;$XlVSM#2Ro*J*mKrAS>l~ zYif}8B2>I)ZtpXJmx?141J@&1vB=U{E?fYku=rWPL^r*_fP|=#|IszVpaXabBJ$bg z+@z!=(rDNZc%8B8^t6^k)%6W;B}0uZdq3Q-n))}V$2<@f?)*#3{l0rQ1dQ6wqj1ok zey4_k7NT(ruq04s0`RLSKYlP*yg*F&7rEa7lGLB%zNXbwd3%OlU|`@|7WMplP~k&t zHOO{$aL6ZPR*8XetvD5nbX14)DFKP|mmmuhlN^v8A>R98oVmB)>aw!3FZ}&2>TLi_ zp@>xMjOB1zHWP1vN|$usE)Wj72W$XNAqo&k6b*;bQxL7xGJn>YSziaP_&W(|X4C<> zlIi|j6w-FZE~9(o1Y(l^7BIp6hWI!zvOvCLwA-E3U*T(59n6O8u*fR}8lX27pCUZR z>StD$H$ZXps(C~72LgU%Fi2AoOu;6FGa_*3&7?aPA3Oi!;{naDeLx%$mz`OQl``K0 z%+U>kC-laD7e;qSrpCwDgCL`D>lULp$knbt2yX#vvHz*{z?VaaF1>m4rb@98 zn|=^S)ss~m)(!pu4_sGjDA2qvdT-t@Oh}vdXq#7n=idsSzes_{ueF0n+qD*(P`R$1 z%98Ck7xl%Y8MI#i;R&8`=eW#H7n++liRZPK^akS-k<50a??cC%*89^=1`{P*R>hSj zYnH=6ro<5L)#xEE^w=esvypMZ+NM8}kK8zCr#NYrjb=}v^-EVVX0767RwVRp>LqGE za%sJPL_HtyTx?ZkA8uY3T!|6!rg#g986-80OlPR)@76^m!(1X6A9fj1Q$crs>SqH5 zx_?DK!7ZRXZ|fh)mEQ`hNn-*jt*<|=X+}uxg&X3->E$z*AWZr>14rDixOrJ`+?o%+Aoki45 z)k|hMJ^k!2rgd0=^tUtV7BT}1{&RKbC}Y<{uB z)DSpZB+M!%DVy(JBGbHv-uZ_?{Fh|nW1tBkTDa=_XHUv54<5@Qj$pj?=D}!R!TgE$ zElo`$d_5~ZUTmvhue%(c?F9QLr^Ea`qGK1XE_GV^uJ;SAUjYtPQ6X@UGA>GCGt1?M z)leRy=Ce5$Q~6DPK?f`xA-vWG8EioQTHeCSM>-+91t!l@?~4m30~orpva)y`=HtMk z;CjpXq|@dL{}uz!;py@%DCQ@+>hN71=!f#T1#$X`Ff&md1H;2r;|mFp5Ly=ZC;t>3 zxfr}v!`wA)+(2y(FxzpU=7YiAKL)I)@xbx=$L!d-=N@?YR)<~J^_t=NG=ZH3kCjW& zY$X#g^~hgN56YrAP|*;y8$py^@HTb2qMmPcQ$r&HZ9U{d9K&QK7L~qk-W+Xjoc-L8 zK6HJQRzPn_0DaL+IVFXFLni$w2&9m@0_mxQT+~>L=KbmQN4~wGdq0*lv*Luww@!?( z`%Zvjv+Z;@^1gEb5rEx!IdRJo)fsf!+k^cbGbNW%?udYJb&v!vEM7wU&R|ND+)J)d zoqv4qnC6xE@9y)U4D5OfAa@24pb++A8Jyr6ved0_sVIp9Pn2})17VM8tiptYho*!x{_yN?< zByHLH(IfpCJ)*C}rAGPnmKKs5BRjzIp$ER{6K3xk9B8Z~TUDl`zh|6DoT-!lqyqmI zXl+wsng!_XC-72RQ9|L&jrifFle`L?_33AO;Idayh75aprAoD@Hlw!@;$ zn6aH+`{@|}1Mhix;cJ%=lASu(2Hgti#y|b)|AxBAmv1Ah`JV0El%Cgff;k?!zW<=o ze(_G<`p(jV4I>1MyC8K01_#HDHpV6bA<}M)5I1r$b^aI2#wI3l8@I|O{PNEe+EJ(DgnqdO%VEykdnTQjc};Mpu&%yQ482sH0tsH z5$c3=aB61J4&^pg1nX@|!4xkvDJelNr8D1qfI0~ZyWV`@y`XrMq5-yIc`p!HKTe6b79G=W zd&wklCR3Fv0bdrHkW9MrA0>HudCAMl`a=;I+BsFwgG+H(oQTiN z8g_I?m#V&c#Ps$LeMbhTUkIpK2l1YrF@GW;NgpB5xOz*_fMDOF+|TQ@oCbCT$FIM- zn>duG1G^fU-anHei`M-1HgoK|H_Mf)x{sg=8Soa+EAkaft#F`;NB|R4oFvc`#b*k@ z;di0Tg+?aMruREJDdayP!;LJxT8y)=e@qBIwm*(nXXEPQ2HAqt>44#LC9Yz;HnmfZ zUsm#Y6K7N(fZiNn^W3dEgi{hynQ^vsq@XjfQBbbN`z=81Jj=pjV&NjH19aLcaSviD zS+GG2E@4qen;&w``ZpOtA^fw=3NR8xd{NIO07Am)mX$}A=-innPwW6l`QP~aeGH_` z&m$vQ#ldLjcz!D2x*!-iWOiSl!4x|ez{&>8`D=#$<;{ddwzLV`)ICD}oy~V#oTvv2 z+}rKo3d-8jXXRkR%DPv70$Fwet~5)s7R7)YXk=V-BmePV$4-b-@Yy8oA)<(xcAj=z z1(7(ZRl`x*S70%Cs$|3EFoX&%bsVE|p9b(|^7tpzb3>3ki!8O?pf4d33h`jS!PP^h zyk|Z>Ez>SbdL3}1iNCW|0K`@~sdY~3vfkl+=SI!8VzH8pN1_{CC6jx|b7Om; zh0mMuoMhCZW6;{U71+#A|BY#&jNXdn<9^A_Jo~HR=JAy4vAg^d>I6TozYATRW&6x{er7#r(&%!* zPUbIlJE~@>gZd0cu48I0D^;40`Gr6jbpBDg7p)NFQYnJYOjM{`w58Ji%|D5c{02)3 z9V@VWaO)zGd}h6FKpSS+PHT7{WxMR3xlJj=+?ixA^Vm#KYTBTSjxpRLNQ9K*+M13)S}KbNnLknrg~P%cgvfm&-#0V zTVRXh6UJm{E!1s=toB6gDWVx)0L{o>zoflS&7cgw-B}A?}Ks0n6S4Bji(h&#N%V(N3L}W8DNpC}Ih}sI<6Y!}d z-wAlvph6mOD>dX4FEA59O9^UDK+}$VYv3)&9|+F9@DrWQ96feDm-VCL8w1CQHZ~x@ z_O@~>;DUeYfYqP_x*KnR;_y?d7lWAiyl1)!Ipn*3;eU{xAN~kMA={&H_0a_Jdlja_-(I@9jvFKsg7-EAjPfVK6Nq zYrcz%%NGH@J%1g24a%^%KOnM&FDHFRpf54nhgu*3i1rciYW?c95O4T!eOchKU0I-8saTEA*WbeA>N%ko0aJK#2U1)c>2Pp!USja{5D8*!zJIG_X zSx=ZXfd`ev;lM$ib(Q;1p)Ck{g>C9{Aae-N)o#n2zY2=9)8ou z_z`aW+QomR*6`4dv~Az)mK_Rd-`?LufBnmA&H@#L$xl)WF;Hcj^zOedmJlFnt?dgo zr)}sa>^XEhesbre{=*=i=XRRb*8_?xs_i!}Fe6R+;gVVnCj0%UFY#+T?=^eFPb zj%hR72&A5_-E6vbwRUX6Z17m8#HDwOM)8;y`C=L*5@$~G+Cv)w{pleUhp6L6J$8=O zVg#6@YuBqT4-4gNP+Ppq4FH#Bh4OU&C!WaW<%+s?*Hi)P?y0?_A;PK`6&TmJPaJ`s z!9T=7^7Gb&OG}mHxAC{CsNc{n44#dC%~HZ`|MrJqPY58GTdG8UeSc!59gHk2Er6_m zf=T`K=_+Jab$~6>*+Ni`@Kaxpt^#aEZq(W~2RqXGwiwK~o8f;4WB;q32)L9c=?&q& zVpCB!==h7da*Fo9^XrCb$nFOfyh{fapxOS zDp?QS-At^k*TL~o3dF|+cJ{#qhb*wmkbQ2;F^vRIB&YN0{GPfl*9{!9nnF`amJ80| zSmWCMoP-1kzzDzW&HqF^p)NdNj6l`^2j_ojS7Oj%)+khl7=I5Z3}Y<*6wG_fzKohA z?Txy;Ff>~=4qTmC-KsTRewr3IRB(1{GZhOmDj2ESGmWLwPkgFSzjDm^^ zg&MT#*|cMk`zZ6yWEB1d!;k;SxT<~DOVpU7)R;;>a$sOUkZm*!IXaHVKa^q0w zaD+_^Xi!2hGPRxy{`YkluB7rzd}I@9fs}vd)TzO!$qN@Q0LqMl3SNKtN1uO8IG^p= zgZsd8kBjO^g$}Tu)s`HUCWukR;QNwzfVQL)BEwV+U~-^Fmjz^wx(UDxjba7r_hZsR zr65F9)Cj;~G4Q1~8!%f#sA+M$lBRRkrB1MR-U8H?4<#fBXVCzP#^Yln8a^HnnrLf7 zjGHxJckPIs(DSJ+?jZq9YySr=4e$VX+@od$Vt{Z0xT#*mfOC}7VWksF2jt9_kM7X@ z7m-xHtIU<=;71_D>-&%t;6% zuSuE*pUpQZd(tRiK7%_{+9F`!nX)TCf-G1))jz(ePx%xtWr;i<&*t zE!620CADqXP@+uHEz-vpT6g*)b~XqWb6o0x2SLOB$znTLlif8kBR$u4Y0Dc*E)#^> zp?7X~aU%QGAC809Zv1^IZtg4PkN@cN{x?nV1KZa9^`ycB{}Y3GSM=*!-O4t)l_s64 zQAELC0v0;S=nrq<{xH6a2|p!3Se%(q3y~RH0tfC_QG%N^g>7vGJ!EXUsM{bgJe_l0STV^fjWw3 zv-JKJLoMLo&L07%V*uCjlqrKDJ0Fx^-*RVlPH55N))KX-$9d@YXJI0(e#elyoYiF2GRpEuU( zHb#K9aR6TiD~4-3Kh^Ghl7AERsMWgLt}Xj|$}L|!haWX7S%qvT!(NOwT; zL!7Kxb$K(mWil59KI)8+kR}{?fQTjt8xV>WG&HD<*tI_3L0D(jEdY*L0GKt@nA}5q z7YHZC6t#Gvjg`99D$Vt%no#cXHy+dt>b6~P!6d6fL!ttmQn2-^+QW9<{vQC%W1}4C zMLwou1x6CCC!=#KneZcW_k*EAcDlQ0;oFr1kE-`y>gZ)FZ!7r1uJU%P`(hQlj3pg? z&wVpNDG5h8`t?NLc|BK_6a0Y^P`TDf6bvRBi zVQhlh*YYJaEXD6k^=xcjteDY$iu;ro)nf8}ZGGi3H0X^Ae@#8NWHwk*Wv#k<7uA;c zeY{jR)JCRk7$>dZFchV>$lr+NP(GZ&JV?NxeCXPw#ScD8QEM@ zDe>LAw)!w~J0D37J?7p4I%C^*ShrB7x%u1ea%1=-9@^W)`~Gl>*D436YM*4F{fQcB zH|5Th3Wc;@YN{0c*@1xT$bksHLnKjoX8&DJ-NhJbbMyBM*CUcZGzX3yIg-FHn{ytR z6t~n1(mUKJrpQ2*f5ucLMGMUqp0u@=#=MMgV?5d+NV*mRZ@v%1@%qU(;bDvBp z>Z_(lC-yKKn0nP$1~M(Q>G+NUUHBqD@AU3KUz=X~&++LPvKA8iWzu8zUZoDlxE?ZYv;j)>k4o^6J}wtA*V{D9D0H z-D7fk`W)mCvR6MWQI%0wpXgi1VxIj>KE}T(p!U%QCCO3~X-`|JYJl|y_vPdrc zy`~j(F!YEjgm?9u*IK~A8%Y^hK_>Cx;}^-Ai3`aU_f8a(^UiR8UEe#pf2M}C#I!tc ziss(2!+xG;&YgRKy81Tl-$a5+Klw0znq$Y}0k&v`mY>royr}$qB|Y78FRN`%=z@=6 zCg?ETj9J~g@$if*p%)VTHof;Ef!;%CQzxnq&i5C*U@8$E$p zAh2g!knXd@1n^f?(@UE@f9}x`-wJG}i8aKTpnnE>WiWt_!M?>J4(dcK{(U!~)RHHV z52#m!YU)BE#BEfPUb=LDY;5dBWF)AIYkrP4e}U}}G|d2ddwyzx<=%(T!{-tcq~Rcr zx(J~oCklAUE}ITx{}@n}!AdI$f*l;p;oI#n&ioS=s_j6W*|NXQ%^mvQ|3ynEWs((d z0BI)S*)zhLU^X=7{Ly!z;HsvSo$CC%UUGxxV&GN4Tv$@yn2upDugT@Z-}+zI`*I?+ zv%Yz=tzO7-{_T=g?pAxW8qX#=f%@^MJ@ogF-(Ua97kRUZ0Q{5#rwcj@36yPT!0vR$ z5K6Y6lW`y~=|^A-2FLgJl@+rEW5u2Mk$R$Xt2%)t_e{n!XTG@;kU+CzEI^#m=c3d% z;aq^4H<=QK73yGVr&;UWJJwTD0zJRhHJq&@_xDoo%R3z2#O1pLEmGj>X9W=!aJo9s zj{4a+bb18HK4;)i6x7lH=3a_fCrOQI<4cC+xPRN^4R(Q^l~n2q=O*I;&&e z-!c6&CKCeH{WJH<+zCp#jB$1)|I25NWiI0I@3sDQoZYmWKjZdiLUxdQ0{r~GMHwS8 z5@fh)PuAjlsBbLP)OlR>9jHP>Q1O&k;Ed;>(d`x3tzSXe6%15Hixi;D z7OpVd#H=UhbYD1h=+Mw;Y$O+r%kmrhiT4#! z(4T7$*L`5DGV4&%pu#r=Ol$#wFwP;6FWCUgod;@}6S_W!;Fu4h>^xxfr4P)?$P$SsP?!U}34P4&!Nb0zu<0Nb3wIPLvJ62^t-li^KaeZRu&1 zKETN~jZwSk5AHi!(qjtEB4lbxva+|qUa&IA>ZHUu>phkrFhl!wMH*p40w-wGY@ zs8cUGnSu$uQZRD24yRp=W4h^p{#W^on3)qui37s+wZ^yp_Q-K zrJS4!p&liyD%SKdqng&ljVUpJoC2cAuGXxqegRO9EjMsWdudvmn_odV$*`OGF4EyT z%-6#PCv)t(6%-L+vOd|fUURfDz130m8^+vK0X7UY0eQBA(aWq08e@!2iWXYTrVW&{ zY&yvuBd5<)U^mu#HVgrZ--gWhaJf&JPqSlk(T z@>@4}%qeaDHg5*dZ|OZZp+|OK_SfJ2tW-V2se~2lh}s7g7Wz*d9u6bc9^AEts7a^9 zrAUf0FS%yW%Ms$pw!YW&3-nWf!vmpsR^x_qRDjkSg%!4MLs*4Tt;83Ua?}K^t23r|Cl5U}mm_>Pg(}&!n!qDIIM^)V zsFnU^w8*0*&0MoL#Va+Ypx?-(maOLk(k**wexAC)1%6+__&$Mba~rOO3Qj2U~WYJ!w*U_Z-v zA4{4u_D?~iMU@qFxPot19{H)@5$3^%I08MSQwJ*?VXhoteP%(hh7sWGaunrKCjlD( zVImXhy1sI!#01!ZT>}?>b*g07KrZBfs>fsT%Q0rEr9mLG!m^FpqrhuXSZb-M0f#*N z@*F7&`C+J+f`qKlQAuF3Y=!0xD`>=BnCw{c@7G|7^|6IRSdi}-b>2Qb9kI`TO5(Vw zdn?qXsKkNRH74E5i-7DNY*2$YA2g*XSSqO34Q!0nfo;ujip}#kUS3|k{qSZGv9&sS z-8BI?BK1-M%UbYVaX>dN7_%@rW-CRP6mV38Y=BNIL}y9H#l_w2;4?1_^@k>>Cr_UA z))i}jNhqLy9DO_bM8OO5R#ot=L5hj$!A;rzM<<@N6KzqLc`KOQ8^8~0HS zd~(B12@GH3Ey*gmW_{PN`i2z+Ss@O_6>#ts7ndT$^dKB|XC|ML!(j>0CXDz9tce3D zu%?Wy4zHqvBD6w5qf4%lzFg9TD}b~^sYVpfh)y?~#^*Zjz+Rxxil zL_w0rmV&_28?uhWhI&vtd^y@!?3=?zzFrHc$r5TD&&7H&Ut57Jv%{VFDlM07_k)b^JL=4#5K zzoucXs{!NIfB>}Dax$wmJFn^oSB`6>%WC{6Y`Avl4ESP0U;Jp%Zb%_q@D{6xk zXgHZ01o#GMn3}mdAsP##@t&pH2mwQCmu2mMpddYPiQfhc(DMpxEv0IXlUc?m;!|2N z=Bxl_3ZcPZ2s+8`f~tKN2vlHp%AG(~EajSo_Ylx zTEPg=AfHkB<~fx*#Y;ugyE|GnA1?ux+yoiW)TKXvp|A#b{Jsx*k8mDB&hsWM`+jfk z2gdWCi-^ade`5~(kD*|7fN}$b9-V6NlU?~9!DCt839tMe*ogpzbW4SDG1JHAHLzs6 ziuxSa|6zH)JQ!H45FoyS;T)DdY`}B%guo$cAwY45LX*xbV2HYt)C;ofvy~8_y>UaU zczq!?G$w`~ZX^ZghE<>RzNZJK?TUt6e1t{&+1`1p<)DLEgbOO1h2bl(DVtV)H=Rwg?f?Dw|7SMV Zh16Oo>HaFqxqxO={F= Date: Sun, 1 Dec 2024 19:17:48 -0800 Subject: [PATCH 05/55] Fix docstring --- package/visualization/plot_pyribs/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package/visualization/plot_pyribs/__init__.py b/package/visualization/plot_pyribs/__init__.py index 94e76b72..6d6630eb 100644 --- a/package/visualization/plot_pyribs/__init__.py +++ b/package/visualization/plot_pyribs/__init__.py @@ -25,7 +25,8 @@ def plot_grid_archive_heatmap( # type: ignore Args: study: Optuna study with a sampler that uses pyribs. This function will plot the result archive from the sampler's scheduler. - ax: Axes on which to plot the heatmap. If None, we create a new axes. + ax: Axes on which to plot the heatmap. If None, we retrieve the current + axes. kwargs: All remaining kwargs will be passed to `grid_archive_heatmap `_. Returns: From 85000537694fa4ff5c20dccab5a508dcb42e92cb Mon Sep 17 00:00:00 2001 From: Bryon Tjanaka Date: Mon, 2 Dec 2024 15:50:50 -0800 Subject: [PATCH 06/55] Add visualization to CmaMaeSampler --- package/samplers/cmamae/README.md | 10 ++++++++++ package/samplers/cmamae/example.py | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/package/samplers/cmamae/README.md b/package/samplers/cmamae/README.md index 7801df9f..52869cca 100644 --- a/package/samplers/cmamae/README.md +++ b/package/samplers/cmamae/README.md @@ -46,12 +46,17 @@ $ pip install ribs ## Example ```python +import matplotlib.pyplot as plt import optuna import optunahub + module = optunahub.load_module("samplers/cmamae") CmaMaeSampler = module.CmaMaeSampler +plot_pyribs = optunahub.load_module(package="visualization/plot_pyribs") +plot_grid_archive_heatmap = plot_pyribs.plot_grid_archive_heatmap + def objective(trial: optuna.trial.Trial) -> float: """Returns an objective followed by two measures.""" @@ -80,6 +85,11 @@ if __name__ == "__main__": ) study = optuna.create_study(sampler=sampler) study.optimize(objective, n_trials=10000) + + fig, ax = plt.subplots(figsize=(8, 6)) + plot_grid_archive_heatmap(study, ax=ax) + plt.savefig("archive.png") + plt.show() ``` ## Others diff --git a/package/samplers/cmamae/example.py b/package/samplers/cmamae/example.py index 2903a81a..97b88743 100644 --- a/package/samplers/cmamae/example.py +++ b/package/samplers/cmamae/example.py @@ -1,3 +1,4 @@ +import matplotlib.pyplot as plt import optuna import optunahub @@ -5,6 +6,9 @@ module = optunahub.load_module("samplers/cmamae") CmaMaeSampler = module.CmaMaeSampler +plot_pyribs = optunahub.load_module(package="visualization/plot_pyribs") +plot_grid_archive_heatmap = plot_pyribs.plot_grid_archive_heatmap + def objective(trial: optuna.trial.Trial) -> float: """Returns an objective followed by two measures.""" @@ -33,3 +37,8 @@ def objective(trial: optuna.trial.Trial) -> float: ) study = optuna.create_study(sampler=sampler) study.optimize(objective, n_trials=10000) + + fig, ax = plt.subplots(figsize=(8, 6)) + plot_grid_archive_heatmap(study, ax=ax) + plt.savefig("archive.png") + plt.show() From 08fd2fb63f0de33273f7a53baf255d5b962e3884 Mon Sep 17 00:00:00 2001 From: Bryon Tjanaka Date: Mon, 2 Dec 2024 15:53:38 -0800 Subject: [PATCH 07/55] Note viz module --- package/samplers/cmamae/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package/samplers/cmamae/README.md b/package/samplers/cmamae/README.md index 52869cca..ac015ef5 100644 --- a/package/samplers/cmamae/README.md +++ b/package/samplers/cmamae/README.md @@ -27,6 +27,9 @@ with improvement ranking, all wrapped up in a However, it is possible to implement many variations of CMA-MAE and other quality diversity algorithms using pyribs. +For visualizing the results of the `CmaMaeSampler`, note that we use the +`plot_grid_archive_heatmap` function from the `plot_pyribs` plugin. + ## Class or Function Names - CmaMaeSampler From 256479c1d87b995ee28a2c29b59b94ef24fff7d1 Mon Sep 17 00:00:00 2001 From: y0z Date: Thu, 5 Dec 2024 16:07:20 +0900 Subject: [PATCH 08/55] Update SMACSampler --- package/samplers/smac_sampler/README.md | 28 +++++++++++++++++++++--- package/samplers/smac_sampler/sampler.py | 12 +++++++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index 460b9caa..db1a4c4c 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -7,9 +7,30 @@ optuna_versions: [3.6.1] license: MIT License --- -## Class or Function Names +## APIs -- SAMCSampler +A sampler that uses SMAC3 v2.2.0. + +Please check the API reference for more details: + +- https://automl.github.io/SMAC3/main/5_api.html + +### `SMACSampler(search_space: dict[str, BaseDistribution], n_trials: int = 100,seed: int | None = None, *,surrogate_model_type: str = "rf", acq_func_type: str = "ei_log", init_design_type: str = "sobol", surrogate_model_rf_num_trees: int = 10, surrogate_model_rf_ratio_features: float = 1.0, surrogate_model_rf_min_samples_split: int = 2, surrogate_model_rf_min_samples_leaf: int = 1, init_design_n_configs: int | None = None, init_design_n_configs_per_hyperparameter: int = 10, init_design_max_ratio: float = 0.25, output_directory: Path = Path("smac3_output"))` + +- `search_space`: A dictionary of Optuna distributions. +- `n_trials`: Number of trials to be evaluated in a study. This argument is used to determine the number of initial configurations by SMAC3. Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. This argument does not have to be precise, but it is better to be exact for better performance. +- `seed`: Seed for random number generator. If `None` is given, seed is generated randomly. +- `surrogate_model_type`: What model to use for the probabilistic model. Either `"gp"` (Gaussian process), `"gp_mcmc"` (Gaussian process with MCMC), or `"rf"` (random forest). Default to `"rf"` (random forest). +- `acq_func_type`: What acquisition function to use. Either `"ei"` (expected improvement), `"ei_log"` (expected improvement with log-scaled function), `"pi"` (probability of improvement), or `"lcb"` (lower confidence bound). Default to `"ei_log"`. +- `init_design_type`: What initialization sampler to use. Either `"sobol"` (Sobol sequence), `"lhd"` (Latin hypercube), or `"random"`. Default to `"sobol"`. +- `surrogate_model_rf_num_trees`: The number of trees used for random forest. Equivalent to `n_estimators` in `RandomForestRegressor` in sklearn. +- `surrogate_model_rf_ratio_features`: The ratio of features to use for each tree training in random forest. Equivalent to `max_features` in `RandomForestRegressor` in sklearn. + `surrogate_model_rf_min_samples_split`: The minimum number of samples required to split an internal node: Equivalent to `min_samples_split` in `RandomForestRegressor` in sklearn. + `surrogate_model_rf_min_samples_leaf`: The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least `min_samples_leaf` training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression. Equivalent to `min_samples_leaf` in `RandomForestRegressor` in sklearn. + `init_design_n_configs`: Number of initial configurations. + `init_design_n_configs_per_hyperparameter`: Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and `n_configs_per_hyperparameter` is set to 10, then 50 initial configurations will be sampled. + `init_design_max_ratio`: Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. Additional configurations are not affected by this parameter. + `output_directy` : `Path`, defaults to `Path("smac3_output")`. The directory in which to save the output. The files are saved in `./output_directory/name/seed`. ## Installation @@ -41,6 +62,7 @@ sampler = SMACSampler( "y": optuna.distributions.IntDistribution(-10, 10), }, n_trials=n_trials, + output_directory="smac3_output", ) study = optuna.create_study(sampler=sampler) study.optimize(objective, n_trials=n_trials) @@ -54,7 +76,7 @@ See [`example.py`](https://github.com/optuna/optunahub-registry/blob/main/packag SMAC is maintained by the SMAC team in [automl.org](https://www.automl.org/). If you have trouble using SMAC, a concrete question or found a bug, please create an issue under the [SMAC](https://github.com/automl/SMAC3) repository. -For all other inquiries, please write an email to smac\[at\]ai\[dot\]uni\[dash\]hannover\[dot\]de. +For all other inquiries, please write an email to smac(at)ai(dot)uni(dash)hannover(dot)de. ### Reference diff --git a/package/samplers/smac_sampler/sampler.py b/package/samplers/smac_sampler/sampler.py index 8c3f412a..09d5966e 100644 --- a/package/samplers/smac_sampler/sampler.py +++ b/package/samplers/smac_sampler/sampler.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Sequence +from pathlib import Path from ConfigSpace import Categorical from ConfigSpace import Configuration @@ -96,6 +97,10 @@ class SMACSampler(optunahub.samplers.SimpleBaseSampler): init_design_max_ratio: Use at most ``n_trials * init_design_max_ratio`` number of configurations in the initial design. Additional configurations are not affected by this parameter. + output_directory: + Path, defaults to Path("smac3_output"). + The directory in which to save the output. + The files are saved in `./output_directory/name/seed`. """ def __init__( @@ -114,11 +119,16 @@ def __init__( init_design_n_configs: int | None = None, init_design_n_configs_per_hyperparameter: int = 10, init_design_max_ratio: float = 0.25, + output_directory: Path = Path("smac3_output"), ) -> None: super().__init__(search_space) self._cs, self._hp_scale_value = self._convert_to_config_space_design_space(search_space) scenario = Scenario( - configspace=self._cs, deterministic=True, n_trials=n_trials, seed=seed or -1 + configspace=self._cs, + deterministic=True, + n_trials=n_trials, + seed=seed or -1, + output_directory=output_directory, ) surrogate_model = self._get_surrogate_model( scenario, From a0c3936c36722aeb295bf3c2945890a8afa8bc42 Mon Sep 17 00:00:00 2001 From: y0z Date: Thu, 5 Dec 2024 16:15:57 +0900 Subject: [PATCH 09/55] Revert change --- package/samplers/smac_sampler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index db1a4c4c..0b978694 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -76,7 +76,7 @@ See [`example.py`](https://github.com/optuna/optunahub-registry/blob/main/packag SMAC is maintained by the SMAC team in [automl.org](https://www.automl.org/). If you have trouble using SMAC, a concrete question or found a bug, please create an issue under the [SMAC](https://github.com/automl/SMAC3) repository. -For all other inquiries, please write an email to smac(at)ai(dot)uni(dash)hannover(dot)de. +For all other inquiries, please write an email to smac\[at\]ai\[dot\]uni\[dash\]hannover\[dot\]de. ### Reference From 0515681baeadfe48ec7612168e596b546dae5392 Mon Sep 17 00:00:00 2001 From: y0z Date: Mon, 9 Dec 2024 15:49:49 +0900 Subject: [PATCH 10/55] Fix message in recipes/README.rst --- recipes/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/README.rst b/recipes/README.rst index 59ef6c9d..96a7a667 100644 --- a/recipes/README.rst +++ b/recipes/README.rst @@ -1,4 +1,4 @@ Tutorial ======== -If you are new to Optuna or want a general introduction, we highly recommend the below video. +Through the following tutorials, you can learn how to develop and register features for OptunaHub. From 790ce84055bdf11e2c5c2b7e5ad04accedf01200 Mon Sep 17 00:00:00 2001 From: y0z Date: Mon, 9 Dec 2024 18:15:24 +0900 Subject: [PATCH 11/55] Add benchmarks tutorials --- docs/source/index.rst | 2 + recipes/002_registration.py | 26 +++++----- recipes/006_benchmarks_basic.py | 81 ++++++++++++++++++++++++++++++ recipes/007_benchmarks_advanced.py | 66 ++++++++++++++++++++++++ 4 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 recipes/006_benchmarks_basic.py create mode 100644 recipes/007_benchmarks_advanced.py diff --git a/docs/source/index.rst b/docs/source/index.rst index c92589e7..7ddbeece 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -14,6 +14,8 @@ Recipes - :doc:`recipes/003_pruner` - :doc:`recipes/004_visualization` - :doc:`recipes/005_debugging` +- :doc:`recipes/006_benchmarks_basic` +- :doc:`recipes/007_benchmarks_advanced` License diff --git a/recipes/002_registration.py b/recipes/002_registration.py index 17ec7abc..aed113a3 100644 --- a/recipes/002_registration.py +++ b/recipes/002_registration.py @@ -1,10 +1,10 @@ """ .. _registration: -How to Register Your Algorithm with OptunaHub +How to Register Your Package with OptunaHub =========================================================== -After implementing your own algorithm, you can register the algorithm as a package with OptunaHub. +After implementing your own algorithm/feature, you can register it as a package with OptunaHub. To add your package to the `optunahub-registry `__ repository, you need to create a pull request from your fork. Your pull request must be aligned with `the contribution guidelines `__. @@ -12,9 +12,9 @@ See the `template directory `__ for an example of the directory structure. | `package `__ -| └── category (e.g. samplers, pruners, and visualization) +| └── category (one of samplers, pruners, visualization, benchmarks) | └── YOUR_PACKAGE_NAME (you need to create this directory and its contents) -| ├── YOUR_ALGORITHM_NAME.py +| ├── YOUR_FEATURE_NAME.py | ├── __init__.py | ├── README.md | ├── LICENSE @@ -24,20 +24,20 @@ | ├── (figure1.png) | └── (numerical_results.png) -An implemented algorithm should be put in the corresponding directory, e.g., a sampler should be put in the ``samplers`` directory. +An implemented feature should be put in the corresponding directory, e.g., a sampler should be put in the ``samplers`` directory. In the ``samplers`` directory, you should create a directory with a unique identifier. -This unique identifier is the name of your algorithm package, is used to load the package, and is unable to change once it is registered. -The package name must be a valid Python module name, preferably one that is easily searchable. +This unique identifier is the name of your package, is used to load the package, and is unable to change once it is registered. +The package name must be a valid Python module name (e.g., please use "_" instead of "-"), preferably one that is easily searchable. Abbreviations are not prohibited in package names, but their abuse should be avoided. The created directory should include the following files: -- ``YOUR_ALGORITHM_NAME.py``: The implementation of your algorithm. -- ``__init__.py``: An initialization file. This file must implement your algorithm or import its implementation from another file, e.g., ``YOUR_ALGORITHM_NAME.py``. -- ``README.md``: A description of your algorithm. This file is used to create an `web page of OptunaHub `_. Let me explain the format of the ``README.md`` file later. -- ``LICENSE``: A license file. This file must contain the license of your algorithm. It should be the MIT license in the alpha version of OptunaHub. -- ``example.py``, ``example.ipynb``: This is optional. This file should contain a simple example of how to use your algorithm (Example: `example.py for Simulated Annealing Sampler `_). You can provide examples in both formats. -- ``requirements.txt``: This is optional. A file that contains the additional dependencies of your algorithm. If there are no additional dependencies other than Optuna and OptunaHub, you do not need to create this file. +- ``YOUR_FEATURE_NAME.py``: The implementation of your feature. +- ``__init__.py``: An initialization file. This file must implement your feature or import its implementation from another file, e.g., ``YOUR_FEATURE_NAME.py``. +- ``README.md``: A description of your package. This file is used to create an `web page of OptunaHub `_. Let me explain the format of the ``README.md`` file later. +- ``LICENSE``: A license file. This file must contain the license of your package. It should be the MIT license in the alpha version of OptunaHub. +- ``example.py``, ``example.ipynb``: This is optional. This file should contain a simple example of how to use your package (Example: `example.py for Simulated Annealing Sampler `_). You can provide examples in both formats. +- ``requirements.txt``: This is optional. A file that contains the additional dependencies of your package. If there are no additional dependencies other than Optuna and OptunaHub, you do not need to create this file. - ``images``: This is optional. A directory that contains images. Only relative references to images in this directory are allowed in README.md, e.g., ``![Numrical Results](images/numerical_results.png)``, and absolute paths to images are not allowed. The first image that appears in README.md will be used as the thumbnail. All files must pass linter and formetter checks to be merged to the optunahub-registry repository. diff --git a/recipes/006_benchmarks_basic.py b/recipes/006_benchmarks_basic.py new file mode 100644 index 00000000..35d7e581 --- /dev/null +++ b/recipes/006_benchmarks_basic.py @@ -0,0 +1,81 @@ +""" +.. _benchmarks_basic: + +How to Implement Your Benchmark Problems with OptunaHub (Basic) +=============================================================== + +OptunaHub provides the ``optunahub.benchmarks`` module for implementing benchmark problems. +In this tutorial, we will explain how to implement your own original benchmark problems using ``optunahub.benchmarks``. +""" + +################################################################################################### +# First of all, import `optuna` and other required modules. +from __future__ import annotations + +import optuna +from optunahub.benchmarks import BaseProblem + + +################################################################################################### +# Next, define your own problem class by inheriting ``BaseProblem`` class. +# Here, let's implement a simple 2-dimensional sphere function. +# +# You need to implement the following methods defined in the ``BaseProblem`` class.: +# +# - ``search_space``: This method returns the dictionary of search space of the problem. Each dictionary element consists of the parameter name and distribution (see `optuna.distributions `__). +# - ``directions``: This method returns the directions of the problem. The return type is the list of `optuna.study.direction `__. +# - ``evaluate``: This method evaluates the objective function by taking the dictionary of input parameters. +class Sphere2D(BaseProblem): + @property + def search_space(self) -> dict[str, optuna.distributions.BaseDistribution]: + return { + "x0": optuna.distributions.FloatDistribution(low=-5, high=5), + "x1": optuna.distributions.FloatDistribution(low=-5, high=5), + } + + @property + def directions(self) -> list[optuna.study.StudyDirection]: + return [optuna.study.StudyDirection.MINIMIZE] + + def evaluate(self, params: dict[str, float]) -> float: + return params["x0"] ** 2 + params["x1"] ** 2 + + +################################################################################################### +# Since ``BaseProblem`` provides the default implementation of ``__call__(optuna.Trial)`` that calls the ``evaluate`` method with the parameters defined in the search space, you can use the problem instance as an objective function for ``study.optimize``. +sphere2d = Sphere2D() +study = optuna.create_study(directions=sphere2d.directions) +study.optimize(sphere2d, n_trials=20) + + +################################################################################################### +# You can also customize the constructor of the problem class and introduce additional attributes. +# The below example implements a variadic-dimensional sphere function. +class SphereND(BaseProblem): + def __init__(self, dim: int) -> None: + self.dim = dim + + @property + def search_space(self) -> dict[str, optuna.distributions.BaseDistribution]: + return { + f"x{i}": optuna.distributions.FloatDistribution(low=-5, high=5) + for i in range(self.dim) + } + + @property + def directions(self) -> list[optuna.study.StudyDirection]: + return [optuna.study.StudyDirection.MINIMIZE] + + def evaluate(self, params: dict[str, float]) -> float: + return sum(params[f"x{i}"] ** 2 for i in range(self.dim)) + + +sphere3d = SphereND(dim=3) +study2 = optuna.create_study(directions=sphere3d.directions) +study2.optimize(sphere3d, n_trials=20) + +################################################################################################### +# After implementing your own pruner, you can register it with OptunaHub. +# See :doc:`002_registration` for how to register your pruner with OptunaHub. +# +# In :ref:`benchmarks_advanced`, how to implement complex benchmark problems such as a problem with dynamic search space are explained. diff --git a/recipes/007_benchmarks_advanced.py b/recipes/007_benchmarks_advanced.py new file mode 100644 index 00000000..405d690d --- /dev/null +++ b/recipes/007_benchmarks_advanced.py @@ -0,0 +1,66 @@ +""" +.. _benchmarks_advanced: + +How to Implement Your Benchmark Problems with OptunaHub (Advanced) +================================================================== + +OptunaHub provides the ``optunahub.benchmarks`` module for implementing benchmark problems. +In this tutorial, we will explain how to implement complex benchmark problems such as a problem with dynamic search space using ``optunahub.benchmarks``. + +For the implementation of simple benchmark problems, please refer to :ref:`benchmarks_basic`. +""" + +################################################################################################### +# Implementing a Problem with Dynamic Search Space +# ------------------------------------------------- +# +# Here, let's implement a problem with a dynamic search space. +# +# First of all, import `optuna` and other required modules. +from __future__ import annotations + +import optuna +from optunahub.benchmarks import BaseProblem + + +################################################################################################### +# Next, define your own problem class by inheriting ``BaseProblem`` class. +# To implement a problem with a dynamic search space, you should override the ``__call__(optuna.Trial)`` method that allows you to define the search space in a define-by-run manner. +# You also need to implement the ``direcitons`` property. +class DynamicProblem(BaseProblem): + def __call__(self, trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -5, 5) + if x < 0: + # Parameter `y` exists only when `x` is negative. + y = trial.suggest_float("y", -5, 5) + return x**2 + y + else: + return x**2 + + @property + def directions(self) -> list[optuna.study.StudyDirection]: + return [optuna.study.StudyDirection.MINIMIZE] + + @property + def search_space(self) -> dict[str, optuna.distributions.BaseDistribution]: + # You can implement this property as you like, or leave it unimplemented (``BaseProblem`` provides this default behavior). + raise NotImplementedError + + def evaluate(self, params: dict[str, float]) -> float: + # You can implement this method as you like, or leave it unimplemented (``BaseProblem`` provides this default behavior). + raise NotImplementedError + + +################################################################################################### +# The implementations of the ``search_space`` and ``evaluate`` are non-trivial when the search space is dynamic. +# However, the ``__call__(optuna.Trial)`` required to run Optuna optimizations no longer depends on the ``evaluate`` method or the ``search_space`` attribute, so you can implement it however you like, or even leave it unimplemented (then, calling them will result in a ``NotImplementedError``). + +################################################################################################### +# Then, you can optimize the problem with Optuna as usual. +dynamic_problem = DynamicProblem() +study = optuna.create_study(directions=dynamic_problem.directions) +study.optimize(dynamic_problem, n_trials=20) + +################################################################################################### +# After implementing your own pruner, you can register it with OptunaHub. +# See :doc:`002_registration` for how to register your pruner with OptunaHub. From 35ca8c9a41bf82d831ed8577fc4cd805227fe175 Mon Sep 17 00:00:00 2001 From: y0z Date: Mon, 9 Dec 2024 19:34:28 +0900 Subject: [PATCH 12/55] Update README --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c10ad1cc..671d56ae 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,11 @@ -# OptunaHub Registry +OptunaHub Registry +================== + +![OptunaHub](https://github.com/user-attachments/assets/ee24b6eb-a431-4e02-ae52-c2538ffe01ee) + +:link: [**OptunaHub**](https://hub.optuna.org/) +| :page_with_curl: [**Docs**](https://optuna.github.io/optunahub-registry/) +| [**Optuna.org**](https://optuna.org/) OptunaHub Registry is a registry service for sharing and discovering user-defined Optuna packages. It provides a platform for users to share their Optuna packages with others and discover useful packages created by other users. @@ -6,6 +13,18 @@ See the [OptunaHub Website](https://hub.optuna.org/) for registered packages. See also the [OptunaHub API documentation](https://optuna.github.io/optunahub/) for the API to use the registry, and the [OptunaHub tutorial](https://optuna.github.io/optunahub-registry/) for how to register and discover packages. +## Contribution + +Any contributions to OptunaHub are more than welcome! + +OptunaHub is composed of the following three related repositories. Please contribute to the appropriate repository for your purposes. +- [optunahub](https://github.com/optuna/optunahub) + - The python library to use OptunaHub. If you find issues and/or bugs in the optunahub library, please report it here via [Github issues](https://github.com/optuna/optunahub/issues). +- [optunahub-registry](https://github.com/optuna/optunahub-registry/) (*this repository*) + - The registry of the OptunaHub packages. If you are interested in registering your package with OptunaHub, please contribute to this repository. For general guidelines on how to contribute to the repository, take a look at [CONTRIBUTING.md](https://github.com/optuna/optunahub-registry/blob/main/CONTRIBUTING.md). +- [optunahub-web](https://github.com/optuna/optunahub-web/) + - The web frontend for OptunaHub. If you find issues and/or bugs on the website, please report it here via [GitHub issues](https://github.com/optuna/optunahub-web/issues). + ## Quick TODO List towards Contribution When creating your package, please check the following TODO list: From add959e4eb0d9267922abc1df57851c610e63d29 Mon Sep 17 00:00:00 2001 From: y0z Date: Mon, 9 Dec 2024 19:37:14 +0900 Subject: [PATCH 13/55] Update menu --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 671d56ae..1581fd80 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ OptunaHub Registry ![OptunaHub](https://github.com/user-attachments/assets/ee24b6eb-a431-4e02-ae52-c2538ffe01ee) :link: [**OptunaHub**](https://hub.optuna.org/) -| :page_with_curl: [**Docs**](https://optuna.github.io/optunahub-registry/) +| :page_with_curl: [**Docs**](https://optuna.github.io/optunahubhub/) +| :page_with_curl: [**Tutorials**](https://optuna.github.io/optunahubhub-registry/) | [**Optuna.org**](https://optuna.org/) OptunaHub Registry is a registry service for sharing and discovering user-defined Optuna packages. It provides a platform for users to share their Optuna packages with others and discover useful packages created by other users. From 0df9763152ceaa25cc5196e3a8d6e65bb1e3267d Mon Sep 17 00:00:00 2001 From: y0z Date: Mon, 9 Dec 2024 19:38:08 +0900 Subject: [PATCH 14/55] Fix URLs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1581fd80..78c06065 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ OptunaHub Registry ![OptunaHub](https://github.com/user-attachments/assets/ee24b6eb-a431-4e02-ae52-c2538ffe01ee) :link: [**OptunaHub**](https://hub.optuna.org/) -| :page_with_curl: [**Docs**](https://optuna.github.io/optunahubhub/) -| :page_with_curl: [**Tutorials**](https://optuna.github.io/optunahubhub-registry/) +| :page_with_curl: [**Docs**](https://optuna.github.io/optunahub/) +| :page_with_curl: [**Tutorials**](https://optuna.github.io/optunahub-registry/) | [**Optuna.org**](https://optuna.org/) OptunaHub Registry is a registry service for sharing and discovering user-defined Optuna packages. It provides a platform for users to share their Optuna packages with others and discover useful packages created by other users. From 991c5fd365b3eff7eabb71861cc0d21f2855e75a Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Tue, 10 Dec 2024 08:49:17 +0100 Subject: [PATCH 15/55] Refactor HEBOSampler based on TPESampler --- package/samplers/hebo/sampler.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/package/samplers/hebo/sampler.py b/package/samplers/hebo/sampler.py index f77a2d2a..46c9e1ce 100644 --- a/package/samplers/hebo/sampler.py +++ b/package/samplers/hebo/sampler.py @@ -85,13 +85,12 @@ def __init__( self._hebo = None self._intersection_search_space = IntersectionSearchSpace() self._independent_sampler = independent_sampler or optuna.samplers.RandomSampler(seed=seed) - self._is_independent_sample_necessary = False self._constant_liar = constant_liar self._rng = np.random.default_rng(seed) def _sample_relative_define_and_run( self, study: Study, trial: FrozenTrial, search_space: dict[str, BaseDistribution] - ) -> dict[str, float]: + ) -> dict[str, Any]: return { name: row.iloc[0] for name, row in self._hebo.suggest().items() @@ -100,7 +99,7 @@ def _sample_relative_define_and_run( def _sample_relative_stateless( self, study: Study, trial: FrozenTrial, search_space: dict[str, BaseDistribution] - ) -> dict[str, float]: + ) -> dict[str, Any]: if self._constant_liar: target_states = [TrialState.COMPLETE, TrialState.RUNNING] else: @@ -113,10 +112,8 @@ def _sample_relative_stateless( # note: The backend HEBO implementation uses Sobol sampling here. # This sampler does not call `hebo.suggest()` here because # Optuna needs to know search space by running the first trial in Define-by-Run. - self._is_independent_sample_necessary = True return {} - else: - self._is_independent_sample_necessary = False + trials = [t for t in trials if set(search_space.keys()) <= set(t.params.keys())] # Assume that the back-end HEBO implementation aims to minimize. @@ -140,7 +137,7 @@ def _sample_relative_stateless( def sample_relative( self, study: Study, trial: FrozenTrial, search_space: dict[str, BaseDistribution] - ) -> dict[str, float]: + ) -> dict[str, Any]: if study._is_multi_objective(): raise ValueError( f"{self.__class__.__name__} has not supported multi-objective optimization." @@ -240,7 +237,9 @@ def sample_independent( param_name: str, param_distribution: BaseDistribution, ) -> Any: - if not self._is_independent_sample_necessary: + states = (TrialState.COMPLETE, TrialState.RUNNING) + trials = study._get_trials(deepcopy=False, states=states, use_cache=True) + if any(param_name in trial.params for trial in trials): warnings.warn( "`HEBOSampler` falls back to `RandomSampler` due to dynamic search space." ) From c9b841961d8dc8cf40149071d8ebf92f1051df5c Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Tue, 10 Dec 2024 08:54:40 +0100 Subject: [PATCH 16/55] Fix --- package/samplers/hebo/sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/hebo/sampler.py b/package/samplers/hebo/sampler.py index 46c9e1ce..ea7c8c0b 100644 --- a/package/samplers/hebo/sampler.py +++ b/package/samplers/hebo/sampler.py @@ -128,7 +128,7 @@ def _sample_relative_stateless( params = pd.DataFrame([t.params for t in trials]) values[np.isnan(values)] = worst_value values *= sign - hebo.observe(params, values) + hebo.observe(params, values[:, np.newaxis]) return { name: row.iloc[0] for name, row in hebo.suggest().items() From 370299176f365c62b4ae3a858c1e778af3b410a8 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Tue, 10 Dec 2024 09:35:51 +0100 Subject: [PATCH 17/55] Replace warning --- package/samplers/hebo/sampler.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/package/samplers/hebo/sampler.py b/package/samplers/hebo/sampler.py index ea7c8c0b..1a5b54b2 100644 --- a/package/samplers/hebo/sampler.py +++ b/package/samplers/hebo/sampler.py @@ -2,7 +2,6 @@ from collections.abc import Sequence from typing import Any -import warnings import numpy as np import optuna @@ -10,6 +9,7 @@ from optuna.distributions import CategoricalDistribution from optuna.distributions import FloatDistribution from optuna.distributions import IntDistribution +from optuna.logging import get_logger from optuna.samplers import BaseSampler from optuna.search_space import IntersectionSearchSpace from optuna.study import Study @@ -23,6 +23,9 @@ from hebo.optimizers.hebo import HEBO +_logger = get_logger(f"optuna.{__name__}") + + class HEBOSampler(optunahub.samplers.SimpleBaseSampler): """A sampler using `HEBO __` as the backend. @@ -240,9 +243,7 @@ def sample_independent( states = (TrialState.COMPLETE, TrialState.RUNNING) trials = study._get_trials(deepcopy=False, states=states, use_cache=True) if any(param_name in trial.params for trial in trials): - warnings.warn( - "`HEBOSampler` falls back to `RandomSampler` due to dynamic search space." - ) + _logger.warn(f"Use `RandomSampler` for {param_name} due to dynamic search space.") return self._independent_sampler.sample_independent( study, trial, param_name, param_distribution From 2be4151d2bed32d2f128898a065149c667c07bf9 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Tue, 10 Dec 2024 18:07:18 +0100 Subject: [PATCH 18/55] Enhance doc --- package/samplers/hebo/README.md | 29 ++++++++++++++++++-------- package/samplers/hebo/requirements.txt | 2 +- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/package/samplers/hebo/README.md b/package/samplers/hebo/README.md index c60ea858..11f1d1a2 100644 --- a/package/samplers/hebo/README.md +++ b/package/samplers/hebo/README.md @@ -14,10 +14,12 @@ license: MIT License ## Installation ```bash -pip install -r https://hub.optuna.org/samplers/hebo/requirements.txt -git clone git@github.com:huawei-noah/HEBO.git -cd HEBO/HEBO -pip install -e . +# Install the dependencies. +pip install optunahub hebo + +# NOTE: Below is optional, but pymoo must be installed after NumPy for faster HEBOSampler, +# we run the following command to make sure that the compiled version is installed. +pip install --upgrade pymoo ``` ## APIs @@ -59,11 +61,7 @@ def objective(trial: optuna.trial.Trial) -> float: module = optunahub.load_module("samplers/hebo") -sampler = module.HEBOSampler(search_space={ - "x": optuna.distributions.FloatDistribution(-10, 10), - "y": optuna.distributions.IntDistribution(-10, 10), -}) -# sampler = module.HEBOSampler() # Note: `search_space` is not required, and thus it works too. +sampler = module.HEBOSampler() study = optuna.create_study(sampler=sampler) study.optimize(objective, n_trials=100) @@ -73,6 +71,19 @@ print(study.best_trial.params, study.best_trial.value) See [`example.py`](https://github.com/optuna/optunahub-registry/blob/main/package/samplers/hebo/example.py) for a full example. ![History Plot](images/hebo_optimization_history.png "History Plot") +Note that it may slightly speed up the sampling routine by giving the search space directly to `HEBOSampler` since Optuna can skip the search space inference. +For example, the instantiation of `HEBOSampler` above can be modified as follows: + +```python +search_space={ + "x": optuna.distributions.FloatDistribution(-10, 10), + "y": optuna.distributions.IntDistribution(-10, 10), +} +sampler = module.HEBOSampler(search_space=search_space) +``` + +However, users need to make sure that the provided search space and the search space defined in the objective function must be consistent. + ## Others HEBO is the winning submission to the [NeurIPS 2020 Black-Box Optimisation Challenge](https://bbochallenge.com/leaderboard). diff --git a/package/samplers/hebo/requirements.txt b/package/samplers/hebo/requirements.txt index 2ba8ec1a..2edc8955 100644 --- a/package/samplers/hebo/requirements.txt +++ b/package/samplers/hebo/requirements.txt @@ -1,3 +1,3 @@ optuna optunahub -hebo@git+https://github.com/huawei-noah/HEBO.git@v0.3.6#subdirectory=HEBO +hebo From 61b55b0395db868bd27b02a9ae7a7dac8c539c41 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 11:07:59 +0900 Subject: [PATCH 19/55] Update recipes/002_registration.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/002_registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/002_registration.py b/recipes/002_registration.py index aed113a3..9c4671a2 100644 --- a/recipes/002_registration.py +++ b/recipes/002_registration.py @@ -4,7 +4,7 @@ How to Register Your Package with OptunaHub =========================================================== -After implementing your own algorithm/feature, you can register it as a package with OptunaHub. +After implementing your own feature, you can register it as a package with OptunaHub. To add your package to the `optunahub-registry `__ repository, you need to create a pull request from your fork. Your pull request must be aligned with `the contribution guidelines `__. From 46b2cdeae0b40ef8ec40c6a3095c64683cec0700 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 11:08:14 +0900 Subject: [PATCH 20/55] Update recipes/002_registration.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/002_registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/002_registration.py b/recipes/002_registration.py index 9c4671a2..f49d201d 100644 --- a/recipes/002_registration.py +++ b/recipes/002_registration.py @@ -12,7 +12,7 @@ See the `template directory `__ for an example of the directory structure. | `package `__ -| └── category (one of samplers, pruners, visualization, benchmarks) +| └── category (samplers, pruners, visualization, or benchmarks) | └── YOUR_PACKAGE_NAME (you need to create this directory and its contents) | ├── YOUR_FEATURE_NAME.py | ├── __init__.py From 3e54e7f7e33ba5e3f36b25beeba8f9086b12396a Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 11:08:35 +0900 Subject: [PATCH 21/55] Update recipes/002_registration.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/002_registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/002_registration.py b/recipes/002_registration.py index f49d201d..ee55ee18 100644 --- a/recipes/002_registration.py +++ b/recipes/002_registration.py @@ -25,7 +25,7 @@ | └── (numerical_results.png) An implemented feature should be put in the corresponding directory, e.g., a sampler should be put in the ``samplers`` directory. -In the ``samplers`` directory, you should create a directory with a unique identifier. +A newly created directory must be named uniquely in the corresponding directory. This unique identifier is the name of your package, is used to load the package, and is unable to change once it is registered. The package name must be a valid Python module name (e.g., please use "_" instead of "-"), preferably one that is easily searchable. Abbreviations are not prohibited in package names, but their abuse should be avoided. From 7e3c23f95145580af06c5ff8aa30b3b22e86906d Mon Sep 17 00:00:00 2001 From: y0z Date: Wed, 11 Dec 2024 11:11:20 +0900 Subject: [PATCH 22/55] Update description about tags --- recipes/002_registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/002_registration.py b/recipes/002_registration.py index aed113a3..4adb641a 100644 --- a/recipes/002_registration.py +++ b/recipes/002_registration.py @@ -74,7 +74,7 @@ - ``author`` (string): The author of the package. It can be your name or your organization name. - ``title`` (string): The package title. It should not be a class/function name but a human-readable name. For example, `Demo Sampler` is a good title, but `DemoSampler` is not. - ``description`` (string): A brief description of the package. It should be a one-sentence summary of the package. - - ``tags`` (list[string]): The package tags. It should be a list of strings. The tags must include ``sampler``, ``visualization``, or ``pruner`` depending on the type of the package. You can add other tags as needed. For example, "['sampler', 'LLM']". + - ``tags`` (list[string]): The package tags. It should be a list of strings. The tags must include ``sampler``, ``visualization``, ``pruner``, or ``benchmarks`` depending on the type of the package. You can add other tags as needed. For example, "['sampler', 'LLM']". - ``optuna_versions`` (list[string]): A list of Optuna versions that the package supports. It should be a list of strings. You can find your Optuna version with ``python -c 'import optuna; print(optuna.__version__)'``. - ``license`` (string): The license of the package. It should be a string. For example, `MIT License`. The license must be `MIT License` in the current version of OptunaHub. From a57ee3b78d06c9cd0bb5279fa26a51c62163d8ff Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:00:29 +0900 Subject: [PATCH 23/55] Update recipes/007_benchmarks_advanced.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/007_benchmarks_advanced.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/007_benchmarks_advanced.py b/recipes/007_benchmarks_advanced.py index 405d690d..2c46ddc3 100644 --- a/recipes/007_benchmarks_advanced.py +++ b/recipes/007_benchmarks_advanced.py @@ -63,4 +63,4 @@ def evaluate(self, params: dict[str, float]) -> float: ################################################################################################### # After implementing your own pruner, you can register it with OptunaHub. -# See :doc:`002_registration` for how to register your pruner with OptunaHub. +# See :doc:`002_registration` for how to register your benchmark problem with OptunaHub. From e3f2bcd78333ca790e4383f00945d95061b49be4 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:00:36 +0900 Subject: [PATCH 24/55] Update recipes/007_benchmarks_advanced.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/007_benchmarks_advanced.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/007_benchmarks_advanced.py b/recipes/007_benchmarks_advanced.py index 2c46ddc3..9757a09b 100644 --- a/recipes/007_benchmarks_advanced.py +++ b/recipes/007_benchmarks_advanced.py @@ -62,5 +62,5 @@ def evaluate(self, params: dict[str, float]) -> float: study.optimize(dynamic_problem, n_trials=20) ################################################################################################### -# After implementing your own pruner, you can register it with OptunaHub. +# After implementing your own benchmark problem, you can register it with OptunaHub. # See :doc:`002_registration` for how to register your benchmark problem with OptunaHub. From ca697494afac2e2d2a52ba87ce0caf14df00b1e3 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:01:03 +0900 Subject: [PATCH 25/55] Update recipes/007_benchmarks_advanced.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/007_benchmarks_advanced.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recipes/007_benchmarks_advanced.py b/recipes/007_benchmarks_advanced.py index 9757a09b..3a5b6e69 100644 --- a/recipes/007_benchmarks_advanced.py +++ b/recipes/007_benchmarks_advanced.py @@ -53,7 +53,9 @@ def evaluate(self, params: dict[str, float]) -> float: ################################################################################################### # The implementations of the ``search_space`` and ``evaluate`` are non-trivial when the search space is dynamic. -# However, the ``__call__(optuna.Trial)`` required to run Optuna optimizations no longer depends on the ``evaluate`` method or the ``search_space`` attribute, so you can implement it however you like, or even leave it unimplemented (then, calling them will result in a ``NotImplementedError``). +# However, since ``__call__(self, trial: optuna.Trial)`` does not have to depend on both the ``evaluate`` method and the ``search_space`` attribute internally, their implementations are up to users. +# If possible, you could provide their implementations, but this is not necessary to make your benchmark problem work. +# Please note that calling them will result in ``NotImplementedError`` if you leave them unimplemented. ################################################################################################### # Then, you can optimize the problem with Optuna as usual. From b2e872ece047b6c3e4d6908b011f4273bfc038dc Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:01:16 +0900 Subject: [PATCH 26/55] Update recipes/007_benchmarks_advanced.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/007_benchmarks_advanced.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/007_benchmarks_advanced.py b/recipes/007_benchmarks_advanced.py index 3a5b6e69..a7b708a5 100644 --- a/recipes/007_benchmarks_advanced.py +++ b/recipes/007_benchmarks_advanced.py @@ -26,7 +26,7 @@ ################################################################################################### # Next, define your own problem class by inheriting ``BaseProblem`` class. # To implement a problem with a dynamic search space, you should override the ``__call__(optuna.Trial)`` method that allows you to define the search space in a define-by-run manner. -# You also need to implement the ``direcitons`` property. +# Please note that ``direcitons`` property must also be implemented. class DynamicProblem(BaseProblem): def __call__(self, trial: optuna.Trial) -> float: x = trial.suggest_float("x", -5, 5) From 2ebeb27576816a8912383601f3d07c133f4367f7 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:01:33 +0900 Subject: [PATCH 27/55] Update recipes/007_benchmarks_advanced.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/007_benchmarks_advanced.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/007_benchmarks_advanced.py b/recipes/007_benchmarks_advanced.py index a7b708a5..a180ad1e 100644 --- a/recipes/007_benchmarks_advanced.py +++ b/recipes/007_benchmarks_advanced.py @@ -25,7 +25,7 @@ ################################################################################################### # Next, define your own problem class by inheriting ``BaseProblem`` class. -# To implement a problem with a dynamic search space, you should override the ``__call__(optuna.Trial)`` method that allows you to define the search space in a define-by-run manner. +# To implement a problem with a dynamic search space, ``__call__(optuna.Trial)`` method must be overridden so that we can define a dynamic search space in the define-by-run manner. # Please note that ``direcitons`` property must also be implemented. class DynamicProblem(BaseProblem): def __call__(self, trial: optuna.Trial) -> float: From fc27bb6d7e8f679ace491f3db8bdae96b9bb9e9b Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:24:56 +0900 Subject: [PATCH 28/55] Update recipes/006_benchmarks_basic.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/006_benchmarks_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/006_benchmarks_basic.py b/recipes/006_benchmarks_basic.py index 35d7e581..7f066db3 100644 --- a/recipes/006_benchmarks_basic.py +++ b/recipes/006_benchmarks_basic.py @@ -5,7 +5,7 @@ =============================================================== OptunaHub provides the ``optunahub.benchmarks`` module for implementing benchmark problems. -In this tutorial, we will explain how to implement your own original benchmark problems using ``optunahub.benchmarks``. +In this tutorial, we will explain how to implement your own benchmark problems using ``optunahub.benchmarks``. """ ################################################################################################### From 04ca437c12e9d288307c5ba8e5f8d3436c38d54d Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:25:05 +0900 Subject: [PATCH 29/55] Update recipes/006_benchmarks_basic.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/006_benchmarks_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/006_benchmarks_basic.py b/recipes/006_benchmarks_basic.py index 7f066db3..e4e9633b 100644 --- a/recipes/006_benchmarks_basic.py +++ b/recipes/006_benchmarks_basic.py @@ -23,7 +23,7 @@ # You need to implement the following methods defined in the ``BaseProblem`` class.: # # - ``search_space``: This method returns the dictionary of search space of the problem. Each dictionary element consists of the parameter name and distribution (see `optuna.distributions `__). -# - ``directions``: This method returns the directions of the problem. The return type is the list of `optuna.study.direction `__. +# - ``directions``: This method returns the directions (minimize or maximize) of the problem. The return type is the list of `optuna.study.direction `__. # - ``evaluate``: This method evaluates the objective function by taking the dictionary of input parameters. class Sphere2D(BaseProblem): @property From 8ce906aba9b955db96fa3934d85015a722fcb143 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:25:14 +0900 Subject: [PATCH 30/55] Update recipes/006_benchmarks_basic.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/006_benchmarks_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/006_benchmarks_basic.py b/recipes/006_benchmarks_basic.py index e4e9633b..6be28905 100644 --- a/recipes/006_benchmarks_basic.py +++ b/recipes/006_benchmarks_basic.py @@ -24,7 +24,7 @@ # # - ``search_space``: This method returns the dictionary of search space of the problem. Each dictionary element consists of the parameter name and distribution (see `optuna.distributions `__). # - ``directions``: This method returns the directions (minimize or maximize) of the problem. The return type is the list of `optuna.study.direction `__. -# - ``evaluate``: This method evaluates the objective function by taking the dictionary of input parameters. +# - ``evaluate``: This method evaluates the objective function given a dictionary of input parameters. class Sphere2D(BaseProblem): @property def search_space(self) -> dict[str, optuna.distributions.BaseDistribution]: From 054e5dc993a558be588743be7f9bf3c1d1c24340 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:25:20 +0900 Subject: [PATCH 31/55] Update recipes/006_benchmarks_basic.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/006_benchmarks_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/006_benchmarks_basic.py b/recipes/006_benchmarks_basic.py index 6be28905..f82ba6d7 100644 --- a/recipes/006_benchmarks_basic.py +++ b/recipes/006_benchmarks_basic.py @@ -42,7 +42,7 @@ def evaluate(self, params: dict[str, float]) -> float: ################################################################################################### -# Since ``BaseProblem`` provides the default implementation of ``__call__(optuna.Trial)`` that calls the ``evaluate`` method with the parameters defined in the search space, you can use the problem instance as an objective function for ``study.optimize``. +# Since ``BaseProblem`` provides the default implementation of ``__call__(self, trial: optuna.Trial)`` that calls the ``evaluate`` method internally, the problem instance can be directly used as an objective function for ``study.optimize``. sphere2d = Sphere2D() study = optuna.create_study(directions=sphere2d.directions) study.optimize(sphere2d, n_trials=20) From 079428cd105344aec6bdafd62ea5be35213b7ecb Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:25:25 +0900 Subject: [PATCH 32/55] Update recipes/006_benchmarks_basic.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/006_benchmarks_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/006_benchmarks_basic.py b/recipes/006_benchmarks_basic.py index f82ba6d7..111ac1b6 100644 --- a/recipes/006_benchmarks_basic.py +++ b/recipes/006_benchmarks_basic.py @@ -49,8 +49,8 @@ def evaluate(self, params: dict[str, float]) -> float: ################################################################################################### -# You can also customize the constructor of the problem class and introduce additional attributes. -# The below example implements a variadic-dimensional sphere function. +# The constructor of the problem class can be customized to introduce additional attributes. +# To give an illustration, we show an example with a dynamic dimensional sphere function. class SphereND(BaseProblem): def __init__(self, dim: int) -> None: self.dim = dim From b2fd00ea97d81fd70b7fa9ed6d1cde51f8bd9570 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:25:31 +0900 Subject: [PATCH 33/55] Update recipes/006_benchmarks_basic.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/006_benchmarks_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/006_benchmarks_basic.py b/recipes/006_benchmarks_basic.py index 111ac1b6..a23bb662 100644 --- a/recipes/006_benchmarks_basic.py +++ b/recipes/006_benchmarks_basic.py @@ -71,8 +71,8 @@ def evaluate(self, params: dict[str, float]) -> float: sphere3d = SphereND(dim=3) -study2 = optuna.create_study(directions=sphere3d.directions) -study2.optimize(sphere3d, n_trials=20) +study = optuna.create_study(directions=sphere3d.directions) +study.optimize(sphere3d, n_trials=20) ################################################################################################### # After implementing your own pruner, you can register it with OptunaHub. From d85aca10e805e9c55ab6ac50761db282e184fe66 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:25:54 +0900 Subject: [PATCH 34/55] Update recipes/006_benchmarks_basic.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/006_benchmarks_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/006_benchmarks_basic.py b/recipes/006_benchmarks_basic.py index a23bb662..31b0637e 100644 --- a/recipes/006_benchmarks_basic.py +++ b/recipes/006_benchmarks_basic.py @@ -75,7 +75,7 @@ def evaluate(self, params: dict[str, float]) -> float: study.optimize(sphere3d, n_trials=20) ################################################################################################### -# After implementing your own pruner, you can register it with OptunaHub. +# After implementing your own benchmark problem, you can register it with OptunaHub. # See :doc:`002_registration` for how to register your pruner with OptunaHub. # # In :ref:`benchmarks_advanced`, how to implement complex benchmark problems such as a problem with dynamic search space are explained. From 7a06cd92aec492f0e2e95cb464a29e00cfb879c2 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:26:00 +0900 Subject: [PATCH 35/55] Update recipes/006_benchmarks_basic.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/006_benchmarks_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/006_benchmarks_basic.py b/recipes/006_benchmarks_basic.py index 31b0637e..c2cdf413 100644 --- a/recipes/006_benchmarks_basic.py +++ b/recipes/006_benchmarks_basic.py @@ -76,6 +76,6 @@ def evaluate(self, params: dict[str, float]) -> float: ################################################################################################### # After implementing your own benchmark problem, you can register it with OptunaHub. -# See :doc:`002_registration` for how to register your pruner with OptunaHub. +# See :doc:`002_registration` for how to register your benchmark problem with OptunaHub. # # In :ref:`benchmarks_advanced`, how to implement complex benchmark problems such as a problem with dynamic search space are explained. From db94241b6104d0a173dfe66a44bf3cf78805e450 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:26:05 +0900 Subject: [PATCH 36/55] Update recipes/006_benchmarks_basic.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- recipes/006_benchmarks_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/006_benchmarks_basic.py b/recipes/006_benchmarks_basic.py index c2cdf413..b443383c 100644 --- a/recipes/006_benchmarks_basic.py +++ b/recipes/006_benchmarks_basic.py @@ -78,4 +78,4 @@ def evaluate(self, params: dict[str, float]) -> float: # After implementing your own benchmark problem, you can register it with OptunaHub. # See :doc:`002_registration` for how to register your benchmark problem with OptunaHub. # -# In :ref:`benchmarks_advanced`, how to implement complex benchmark problems such as a problem with dynamic search space are explained. +# In :ref:`benchmarks_advanced`, we explain how to implement more complex benchmark problems such as a problem with dynamic search space. From 9c99d2e32e1ed0a2274cb80507b4c2a4e6ed45d8 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:58:23 +0900 Subject: [PATCH 37/55] Update package/samplers/smac_sampler/README.md Co-authored-by: Naoto Mizuno --- package/samplers/smac_sampler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index 0b978694..ddc28a26 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -15,7 +15,7 @@ Please check the API reference for more details: - https://automl.github.io/SMAC3/main/5_api.html -### `SMACSampler(search_space: dict[str, BaseDistribution], n_trials: int = 100,seed: int | None = None, *,surrogate_model_type: str = "rf", acq_func_type: str = "ei_log", init_design_type: str = "sobol", surrogate_model_rf_num_trees: int = 10, surrogate_model_rf_ratio_features: float = 1.0, surrogate_model_rf_min_samples_split: int = 2, surrogate_model_rf_min_samples_leaf: int = 1, init_design_n_configs: int | None = None, init_design_n_configs_per_hyperparameter: int = 10, init_design_max_ratio: float = 0.25, output_directory: Path = Path("smac3_output"))` +### `SMACSampler(search_space: dict[str, BaseDistribution], n_trials: int = 100, seed: int | None = None, *, surrogate_model_type: str = "rf", acq_func_type: str = "ei_log", init_design_type: str = "sobol", surrogate_model_rf_num_trees: int = 10, surrogate_model_rf_ratio_features: float = 1.0, surrogate_model_rf_min_samples_split: int = 2, surrogate_model_rf_min_samples_leaf: int = 1, init_design_n_configs: int | None = None, init_design_n_configs_per_hyperparameter: int = 10, init_design_max_ratio: float = 0.25, output_directory: Path = Path("smac3_output"))` - `search_space`: A dictionary of Optuna distributions. - `n_trials`: Number of trials to be evaluated in a study. This argument is used to determine the number of initial configurations by SMAC3. Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. This argument does not have to be precise, but it is better to be exact for better performance. From e3b3330fc1597280566f7398fa6cca156c977977 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:59:50 +0900 Subject: [PATCH 38/55] Update package/samplers/smac_sampler/README.md Co-authored-by: Naoto Mizuno --- package/samplers/smac_sampler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index ddc28a26..2df6e98c 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -25,7 +25,7 @@ Please check the API reference for more details: - `init_design_type`: What initialization sampler to use. Either `"sobol"` (Sobol sequence), `"lhd"` (Latin hypercube), or `"random"`. Default to `"sobol"`. - `surrogate_model_rf_num_trees`: The number of trees used for random forest. Equivalent to `n_estimators` in `RandomForestRegressor` in sklearn. - `surrogate_model_rf_ratio_features`: The ratio of features to use for each tree training in random forest. Equivalent to `max_features` in `RandomForestRegressor` in sklearn. - `surrogate_model_rf_min_samples_split`: The minimum number of samples required to split an internal node: Equivalent to `min_samples_split` in `RandomForestRegressor` in sklearn. +- `surrogate_model_rf_min_samples_split`: The minimum number of samples required to split an internal node: Equivalent to `min_samples_split` in `RandomForestRegressor` in sklearn. `surrogate_model_rf_min_samples_leaf`: The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least `min_samples_leaf` training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression. Equivalent to `min_samples_leaf` in `RandomForestRegressor` in sklearn. `init_design_n_configs`: Number of initial configurations. `init_design_n_configs_per_hyperparameter`: Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and `n_configs_per_hyperparameter` is set to 10, then 50 initial configurations will be sampled. From 98a5541d7d3fd79f1c4f7e0a5c252ec8cf74c32d Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:00:01 +0900 Subject: [PATCH 39/55] Update package/samplers/smac_sampler/README.md Co-authored-by: Naoto Mizuno --- package/samplers/smac_sampler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index 2df6e98c..ec5953d8 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -26,7 +26,7 @@ Please check the API reference for more details: - `surrogate_model_rf_num_trees`: The number of trees used for random forest. Equivalent to `n_estimators` in `RandomForestRegressor` in sklearn. - `surrogate_model_rf_ratio_features`: The ratio of features to use for each tree training in random forest. Equivalent to `max_features` in `RandomForestRegressor` in sklearn. - `surrogate_model_rf_min_samples_split`: The minimum number of samples required to split an internal node: Equivalent to `min_samples_split` in `RandomForestRegressor` in sklearn. - `surrogate_model_rf_min_samples_leaf`: The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least `min_samples_leaf` training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression. Equivalent to `min_samples_leaf` in `RandomForestRegressor` in sklearn. +- `surrogate_model_rf_min_samples_leaf`: The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least `min_samples_leaf` training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression. Equivalent to `min_samples_leaf` in `RandomForestRegressor` in sklearn. `init_design_n_configs`: Number of initial configurations. `init_design_n_configs_per_hyperparameter`: Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and `n_configs_per_hyperparameter` is set to 10, then 50 initial configurations will be sampled. `init_design_max_ratio`: Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. Additional configurations are not affected by this parameter. From 070dbbd03261942d32af97e8223437a3c513c5a2 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:00:19 +0900 Subject: [PATCH 40/55] Update package/samplers/smac_sampler/README.md Co-authored-by: Naoto Mizuno --- package/samplers/smac_sampler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index ec5953d8..f0d6b359 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -27,7 +27,7 @@ Please check the API reference for more details: - `surrogate_model_rf_ratio_features`: The ratio of features to use for each tree training in random forest. Equivalent to `max_features` in `RandomForestRegressor` in sklearn. - `surrogate_model_rf_min_samples_split`: The minimum number of samples required to split an internal node: Equivalent to `min_samples_split` in `RandomForestRegressor` in sklearn. - `surrogate_model_rf_min_samples_leaf`: The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least `min_samples_leaf` training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression. Equivalent to `min_samples_leaf` in `RandomForestRegressor` in sklearn. - `init_design_n_configs`: Number of initial configurations. +- `init_design_n_configs`: Number of initial configurations. `init_design_n_configs_per_hyperparameter`: Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and `n_configs_per_hyperparameter` is set to 10, then 50 initial configurations will be sampled. `init_design_max_ratio`: Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. Additional configurations are not affected by this parameter. `output_directy` : `Path`, defaults to `Path("smac3_output")`. The directory in which to save the output. The files are saved in `./output_directory/name/seed`. From 75301da70d7657df56f4cac6eb04cbf7c4dfea66 Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:00:26 +0900 Subject: [PATCH 41/55] Update package/samplers/smac_sampler/README.md Co-authored-by: Naoto Mizuno --- package/samplers/smac_sampler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index f0d6b359..579e3c47 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -28,7 +28,7 @@ Please check the API reference for more details: - `surrogate_model_rf_min_samples_split`: The minimum number of samples required to split an internal node: Equivalent to `min_samples_split` in `RandomForestRegressor` in sklearn. - `surrogate_model_rf_min_samples_leaf`: The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least `min_samples_leaf` training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression. Equivalent to `min_samples_leaf` in `RandomForestRegressor` in sklearn. - `init_design_n_configs`: Number of initial configurations. - `init_design_n_configs_per_hyperparameter`: Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and `n_configs_per_hyperparameter` is set to 10, then 50 initial configurations will be sampled. +- `init_design_n_configs_per_hyperparameter`: Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and `n_configs_per_hyperparameter` is set to 10, then 50 initial configurations will be sampled. `init_design_max_ratio`: Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. Additional configurations are not affected by this parameter. `output_directy` : `Path`, defaults to `Path("smac3_output")`. The directory in which to save the output. The files are saved in `./output_directory/name/seed`. From 4940b0e29d6526a8b29dd88439b4b9db6b77073d Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:00:32 +0900 Subject: [PATCH 42/55] Update package/samplers/smac_sampler/README.md Co-authored-by: Naoto Mizuno --- package/samplers/smac_sampler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index 579e3c47..476b8d4e 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -29,7 +29,7 @@ Please check the API reference for more details: - `surrogate_model_rf_min_samples_leaf`: The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least `min_samples_leaf` training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression. Equivalent to `min_samples_leaf` in `RandomForestRegressor` in sklearn. - `init_design_n_configs`: Number of initial configurations. - `init_design_n_configs_per_hyperparameter`: Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and `n_configs_per_hyperparameter` is set to 10, then 50 initial configurations will be sampled. - `init_design_max_ratio`: Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. Additional configurations are not affected by this parameter. +- `init_design_max_ratio`: Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. Additional configurations are not affected by this parameter. `output_directy` : `Path`, defaults to `Path("smac3_output")`. The directory in which to save the output. The files are saved in `./output_directory/name/seed`. ## Installation From 19bd153965f9b49e1a92cd9d0232b05b4992757e Mon Sep 17 00:00:00 2001 From: Yoshihiko Ozaki <30489874+y0z@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:00:40 +0900 Subject: [PATCH 43/55] Update package/samplers/smac_sampler/README.md Co-authored-by: Naoto Mizuno --- package/samplers/smac_sampler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index 476b8d4e..ebd2d853 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -30,7 +30,7 @@ Please check the API reference for more details: - `init_design_n_configs`: Number of initial configurations. - `init_design_n_configs_per_hyperparameter`: Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and `n_configs_per_hyperparameter` is set to 10, then 50 initial configurations will be sampled. - `init_design_max_ratio`: Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. Additional configurations are not affected by this parameter. - `output_directy` : `Path`, defaults to `Path("smac3_output")`. The directory in which to save the output. The files are saved in `./output_directory/name/seed`. +- `output_directy` : `Path`, defaults to `Path("smac3_output")`. The directory in which to save the output. The files are saved in `./output_directory/name/seed`. ## Installation From b9c942069c432e1a32b66daacfe2375640b059c6 Mon Sep 17 00:00:00 2001 From: y0z Date: Wed, 11 Dec 2024 17:06:17 +0900 Subject: [PATCH 44/55] Update Path API --- package/samplers/smac_sampler/README.md | 2 +- package/samplers/smac_sampler/sampler.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index ebd2d853..dc9e796a 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -30,7 +30,7 @@ Please check the API reference for more details: - `init_design_n_configs`: Number of initial configurations. - `init_design_n_configs_per_hyperparameter`: Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and `n_configs_per_hyperparameter` is set to 10, then 50 initial configurations will be sampled. - `init_design_max_ratio`: Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. Additional configurations are not affected by this parameter. -- `output_directy` : `Path`, defaults to `Path("smac3_output")`. The directory in which to save the output. The files are saved in `./output_directory/name/seed`. +- `output_directy`: Output directory path, defaults to `"smac3_output"`. The directory in which to save the output. The files are saved in `./output_directory/name/seed`. ## Installation diff --git a/package/samplers/smac_sampler/sampler.py b/package/samplers/smac_sampler/sampler.py index 09d5966e..a92179b2 100644 --- a/package/samplers/smac_sampler/sampler.py +++ b/package/samplers/smac_sampler/sampler.py @@ -98,7 +98,7 @@ class SMACSampler(optunahub.samplers.SimpleBaseSampler): Use at most ``n_trials * init_design_max_ratio`` number of configurations in the initial design. Additional configurations are not affected by this parameter. output_directory: - Path, defaults to Path("smac3_output"). + Output directory path, defaults to "smac3_output". The directory in which to save the output. The files are saved in `./output_directory/name/seed`. """ @@ -119,7 +119,7 @@ def __init__( init_design_n_configs: int | None = None, init_design_n_configs_per_hyperparameter: int = 10, init_design_max_ratio: float = 0.25, - output_directory: Path = Path("smac3_output"), + output_directory: str = "smac3_output", ) -> None: super().__init__(search_space) self._cs, self._hp_scale_value = self._convert_to_config_space_design_space(search_space) @@ -128,7 +128,7 @@ def __init__( deterministic=True, n_trials=n_trials, seed=seed or -1, - output_directory=output_directory, + output_directory=Path(output_directory), ) surrogate_model = self._get_surrogate_model( scenario, From 7e5e8b1d9576e1f1eb92f3066e2adc393c8b7453 Mon Sep 17 00:00:00 2001 From: y0z Date: Wed, 11 Dec 2024 17:07:11 +0900 Subject: [PATCH 45/55] Fix --- package/samplers/smac_sampler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/smac_sampler/README.md b/package/samplers/smac_sampler/README.md index dc9e796a..50882ec9 100644 --- a/package/samplers/smac_sampler/README.md +++ b/package/samplers/smac_sampler/README.md @@ -15,7 +15,7 @@ Please check the API reference for more details: - https://automl.github.io/SMAC3/main/5_api.html -### `SMACSampler(search_space: dict[str, BaseDistribution], n_trials: int = 100, seed: int | None = None, *, surrogate_model_type: str = "rf", acq_func_type: str = "ei_log", init_design_type: str = "sobol", surrogate_model_rf_num_trees: int = 10, surrogate_model_rf_ratio_features: float = 1.0, surrogate_model_rf_min_samples_split: int = 2, surrogate_model_rf_min_samples_leaf: int = 1, init_design_n_configs: int | None = None, init_design_n_configs_per_hyperparameter: int = 10, init_design_max_ratio: float = 0.25, output_directory: Path = Path("smac3_output"))` +### `SMACSampler(search_space: dict[str, BaseDistribution], n_trials: int = 100, seed: int | None = None, *, surrogate_model_type: str = "rf", acq_func_type: str = "ei_log", init_design_type: str = "sobol", surrogate_model_rf_num_trees: int = 10, surrogate_model_rf_ratio_features: float = 1.0, surrogate_model_rf_min_samples_split: int = 2, surrogate_model_rf_min_samples_leaf: int = 1, init_design_n_configs: int | None = None, init_design_n_configs_per_hyperparameter: int = 10, init_design_max_ratio: float = 0.25, output_directory: str = "smac3_output")` - `search_space`: A dictionary of Optuna distributions. - `n_trials`: Number of trials to be evaluated in a study. This argument is used to determine the number of initial configurations by SMAC3. Use at most `n_trials * init_design_max_ratio` number of configurations in the initial design. This argument does not have to be precise, but it is better to be exact for better performance. From f70980e1fb8713f18ec0d376b558dd341f21d5ee Mon Sep 17 00:00:00 2001 From: y0z Date: Wed, 11 Dec 2024 17:44:59 +0900 Subject: [PATCH 46/55] Update --- recipes/002_registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/002_registration.py b/recipes/002_registration.py index e82b3ba4..3e554714 100644 --- a/recipes/002_registration.py +++ b/recipes/002_registration.py @@ -74,7 +74,7 @@ - ``author`` (string): The author of the package. It can be your name or your organization name. - ``title`` (string): The package title. It should not be a class/function name but a human-readable name. For example, `Demo Sampler` is a good title, but `DemoSampler` is not. - ``description`` (string): A brief description of the package. It should be a one-sentence summary of the package. - - ``tags`` (list[string]): The package tags. It should be a list of strings. The tags must include ``sampler``, ``visualization``, ``pruner``, or ``benchmarks`` depending on the type of the package. You can add other tags as needed. For example, "['sampler', 'LLM']". + - ``tags`` (list[string]): The package tags. It should be a list of strings. The tags must include ``sampler``, ``visualization``, ``pruner``, or ``benchmark`` depending on the type of the package. You can add other tags as needed. For example, "['sampler', 'LLM']". - ``optuna_versions`` (list[string]): A list of Optuna versions that the package supports. It should be a list of strings. You can find your Optuna version with ``python -c 'import optuna; print(optuna.__version__)'``. - ``license`` (string): The license of the package. It should be a string. For example, `MIT License`. The license must be `MIT License` in the current version of OptunaHub. From 8fc84e9b85548e4e1bad3341884ce6ebd1be16ea Mon Sep 17 00:00:00 2001 From: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Date: Thu, 12 Dec 2024 00:11:18 +0900 Subject: [PATCH 47/55] Update package/samplers/hebo/sampler.py --- package/samplers/hebo/sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/hebo/sampler.py b/package/samplers/hebo/sampler.py index 1a5b54b2..0ae5c94f 100644 --- a/package/samplers/hebo/sampler.py +++ b/package/samplers/hebo/sampler.py @@ -240,7 +240,7 @@ def sample_independent( param_name: str, param_distribution: BaseDistribution, ) -> Any: - states = (TrialState.COMPLETE, TrialState.RUNNING) + states = (TrialState.COMPLETE, ) trials = study._get_trials(deepcopy=False, states=states, use_cache=True) if any(param_name in trial.params for trial in trials): _logger.warn(f"Use `RandomSampler` for {param_name} due to dynamic search space.") From 17878b4030154b4c79deec7c48811ea80f4b7aaf Mon Sep 17 00:00:00 2001 From: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Date: Thu, 12 Dec 2024 03:25:00 +0100 Subject: [PATCH 48/55] Update package/visualization/plot_pyribs/README.md --- package/visualization/plot_pyribs/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/package/visualization/plot_pyribs/README.md b/package/visualization/plot_pyribs/README.md index 6825fff1..36beb444 100644 --- a/package/visualization/plot_pyribs/README.md +++ b/package/visualization/plot_pyribs/README.md @@ -9,7 +9,13 @@ license: MIT License ## Class or Function Names -- plot_grid_archive_heatmap +- `plot_grid_archive_heatmap()` + +- `HEBOSampler(study: optuna.Study, ax: plt.Axes, **kwargs)` + - `study`: Optuna study with a sampler that uses pyribs. This function will plot the result archive from the sampler's scheduler. + - ax: Axes on which to plot the heatmap. If None, we retrieve the current axes. + - kwargs: All remaining kwargs will be passed to [`grid_archive_heatmap`](https://docs.pyribs.org/en/stable/api/ribs.visualize.grid_archive_heatmap.html). + ## Installation From e289417fef194e3ccdeea369f1c7e8add3548e8a Mon Sep 17 00:00:00 2001 From: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Date: Thu, 12 Dec 2024 03:29:13 +0100 Subject: [PATCH 49/55] Update package/visualization/plot_pyribs/README.md --- package/visualization/plot_pyribs/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package/visualization/plot_pyribs/README.md b/package/visualization/plot_pyribs/README.md index 36beb444..308cb702 100644 --- a/package/visualization/plot_pyribs/README.md +++ b/package/visualization/plot_pyribs/README.md @@ -11,10 +11,10 @@ license: MIT License - `plot_grid_archive_heatmap()` -- `HEBOSampler(study: optuna.Study, ax: plt.Axes, **kwargs)` +- `plot_grid_archive_heatmap(study: optuna.Study, ax: plt.Axes, **kwargs)` - `study`: Optuna study with a sampler that uses pyribs. This function will plot the result archive from the sampler's scheduler. - - ax: Axes on which to plot the heatmap. If None, we retrieve the current axes. - - kwargs: All remaining kwargs will be passed to [`grid_archive_heatmap`](https://docs.pyribs.org/en/stable/api/ribs.visualize.grid_archive_heatmap.html). + - `ax`: Axes on which to plot the heatmap. If None, we retrieve the current axes. + - `**kwargs`: All remaining kwargs will be passed to [`grid_archive_heatmap`](https://docs.pyribs.org/en/stable/api/ribs.visualize.grid_archive_heatmap.html). ## Installation From e519d362ac6cf29413c516646d36c6766267fc03 Mon Sep 17 00:00:00 2001 From: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Date: Thu, 12 Dec 2024 03:29:59 +0100 Subject: [PATCH 50/55] Update package/visualization/plot_pyribs/README.md --- package/visualization/plot_pyribs/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/package/visualization/plot_pyribs/README.md b/package/visualization/plot_pyribs/README.md index 308cb702..1f2e1309 100644 --- a/package/visualization/plot_pyribs/README.md +++ b/package/visualization/plot_pyribs/README.md @@ -9,8 +9,6 @@ license: MIT License ## Class or Function Names -- `plot_grid_archive_heatmap()` - - `plot_grid_archive_heatmap(study: optuna.Study, ax: plt.Axes, **kwargs)` - `study`: Optuna study with a sampler that uses pyribs. This function will plot the result archive from the sampler's scheduler. - `ax`: Axes on which to plot the heatmap. If None, we retrieve the current axes. From 43e2e0abfc9e0d367a3a6e18f0b71e88b003ec54 Mon Sep 17 00:00:00 2001 From: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Date: Thu, 12 Dec 2024 03:30:53 +0100 Subject: [PATCH 51/55] Fix pre-commit --- package/visualization/plot_pyribs/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/package/visualization/plot_pyribs/README.md b/package/visualization/plot_pyribs/README.md index 1f2e1309..7ffd9aae 100644 --- a/package/visualization/plot_pyribs/README.md +++ b/package/visualization/plot_pyribs/README.md @@ -10,6 +10,7 @@ license: MIT License ## Class or Function Names - `plot_grid_archive_heatmap(study: optuna.Study, ax: plt.Axes, **kwargs)` + - `study`: Optuna study with a sampler that uses pyribs. This function will plot the result archive from the sampler's scheduler. - `ax`: Axes on which to plot the heatmap. If None, we retrieve the current axes. - `**kwargs`: All remaining kwargs will be passed to [`grid_archive_heatmap`](https://docs.pyribs.org/en/stable/api/ribs.visualize.grid_archive_heatmap.html). From 48e1bf4082a8e4cab6a42ffa9ceac135bac3c185 Mon Sep 17 00:00:00 2001 From: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Date: Thu, 12 Dec 2024 03:32:16 +0100 Subject: [PATCH 52/55] Fix pre-commit --- package/visualization/plot_pyribs/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/package/visualization/plot_pyribs/README.md b/package/visualization/plot_pyribs/README.md index 7ffd9aae..f7eb29ca 100644 --- a/package/visualization/plot_pyribs/README.md +++ b/package/visualization/plot_pyribs/README.md @@ -15,7 +15,6 @@ license: MIT License - `ax`: Axes on which to plot the heatmap. If None, we retrieve the current axes. - `**kwargs`: All remaining kwargs will be passed to [`grid_archive_heatmap`](https://docs.pyribs.org/en/stable/api/ribs.visualize.grid_archive_heatmap.html). - ## Installation ```shell From 087d11aa5305ff941b2d551c738c1f99739e6b74 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Thu, 12 Dec 2024 03:35:32 +0100 Subject: [PATCH 53/55] Fix --- package/samplers/hebo/sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/samplers/hebo/sampler.py b/package/samplers/hebo/sampler.py index 0ae5c94f..c9ddc3c9 100644 --- a/package/samplers/hebo/sampler.py +++ b/package/samplers/hebo/sampler.py @@ -240,7 +240,7 @@ def sample_independent( param_name: str, param_distribution: BaseDistribution, ) -> Any: - states = (TrialState.COMPLETE, ) + states = (TrialState.COMPLETE,) trials = study._get_trials(deepcopy=False, states=states, use_cache=True) if any(param_name in trial.params for trial in trials): _logger.warn(f"Use `RandomSampler` for {param_name} due to dynamic search space.") From ba4c0400e46280f4fe4acc57f27a8d01f11e8d77 Mon Sep 17 00:00:00 2001 From: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Date: Thu, 12 Dec 2024 03:40:05 +0100 Subject: [PATCH 54/55] Update recipes/007_benchmarks_advanced.py --- recipes/007_benchmarks_advanced.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/007_benchmarks_advanced.py b/recipes/007_benchmarks_advanced.py index a180ad1e..464cb9bd 100644 --- a/recipes/007_benchmarks_advanced.py +++ b/recipes/007_benchmarks_advanced.py @@ -25,7 +25,7 @@ ################################################################################################### # Next, define your own problem class by inheriting ``BaseProblem`` class. -# To implement a problem with a dynamic search space, ``__call__(optuna.Trial)`` method must be overridden so that we can define a dynamic search space in the define-by-run manner. +# To implement a problem with a dynamic search space, ``__call__(self, trial: optuna.Trial)`` method must be overridden so that we can define a dynamic search space in the define-by-run manner. # Please note that ``direcitons`` property must also be implemented. class DynamicProblem(BaseProblem): def __call__(self, trial: optuna.Trial) -> float: From b868bf0099cbcd7d81271c6a27569ee66defa96f Mon Sep 17 00:00:00 2001 From: Bryon Tjanaka Date: Thu, 12 Dec 2024 08:43:48 -0800 Subject: [PATCH 55/55] Link to other plugins in READMEs --- package/samplers/cmamae/README.md | 3 ++- package/visualization/plot_pyribs/README.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/package/samplers/cmamae/README.md b/package/samplers/cmamae/README.md index ac015ef5..b14a5d17 100644 --- a/package/samplers/cmamae/README.md +++ b/package/samplers/cmamae/README.md @@ -28,7 +28,8 @@ However, it is possible to implement many variations of CMA-MAE and other quality diversity algorithms using pyribs. For visualizing the results of the `CmaMaeSampler`, note that we use the -`plot_grid_archive_heatmap` function from the `plot_pyribs` plugin. +`plot_grid_archive_heatmap` function from the +[`plot_pyribs`](https://hub.optuna.org/visualization/plot_pyribs/) plugin. ## Class or Function Names diff --git a/package/visualization/plot_pyribs/README.md b/package/visualization/plot_pyribs/README.md index f7eb29ca..979524e3 100644 --- a/package/visualization/plot_pyribs/README.md +++ b/package/visualization/plot_pyribs/README.md @@ -1,7 +1,7 @@ --- author: Bryon Tjanaka title: Pyribs Visualization Wrappers -description: This visualizaton module provides wrappers around the visualization functions from pyribs, which is useful for plotting results from CmaMaeSampler. +description: This visualizaton module provides wrappers around the visualization functions from pyribs, which is useful for plotting results from [CmaMaeSampler](https://hub.optuna.org/samplers/cmamae/). tags: [visualization, quality diversity, pyribs] optuna_versions: [4.0.0] license: MIT License