From b0cf07ff9d0d1aa80a73d2f902c63b28cc6f60d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 2 Jul 2024 09:30:29 +0200 Subject: [PATCH 01/97] chore: run CI tests when targeting `v2` branch (#376) --- .github/workflows/elixir.yml | 114 +++++++++-------------------------- VERSION | 2 +- mix.lock | 1 + 3 files changed, 32 insertions(+), 85 deletions(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 7c9c7352..cce7e1ff 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -3,6 +3,7 @@ on: pull_request: branches: - main + - v2 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -37,49 +38,6 @@ jobs: - name: Install dependencies run: mix deps.get - compile: - name: Compile project in test env - runs-on: u22-arm-runner - needs: [deps] - - steps: - - uses: actions/checkout@v4 - - name: Setup Elixir - id: beam - uses: erlef/setup-beam@v1 - with: - otp-version: '25.3.2.7' - elixir-version: '1.14.5' - - name: Cache Mix - uses: actions/cache@v4 - with: - path: deps - key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - restore-keys: | - ${{ runner.os }}-mix- - - name: Cache Build - uses: actions/cache@v4 - with: - path: | - _build/${{ env.MIX_ENV }} - key: ${{ runner.os }}-build-${{ env.MIX_ENV }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-build-${{ env.MIX_ENV }}- - - name: Cache native - uses: actions/cache@v4 - id: native-cache - with: - path: | - priv/native - key: ${{ runner.os }}-build-native-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-${{ hashFiles(format('{0}{1}', github.workspace, '/native/**/*')) }} - - name: Set up Rust - uses: dtolnay/rust-toolchain@v1 - if: steps.native-cache.outputs.cache-hit != 'true' || steps.elixir-cache.output.cache-hit != 'true' - with: - toolchain: stable - - name: Compile - run: mix compile - format: name: Formatting checks runs-on: u22-arm-runner @@ -98,15 +56,13 @@ jobs: with: path: deps key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - restore-keys: | - ${{ runner.os }}-mix- - name: Run format check run: mix format --check-formatted credo: name: Code style runs-on: u22-arm-runner - needs: [compile] + needs: [deps] steps: - uses: actions/checkout@v4 @@ -121,23 +77,15 @@ jobs: with: path: deps key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - restore-keys: | - ${{ runner.os }}-mix- - - name: Cache Build - uses: actions/cache@v4 - with: - path: | - _build/${{ env.MIX_ENV }} - key: ${{ runner.os }}-build-${{ env.MIX_ENV }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-build-${{ env.MIX_ENV }}- + - name: Compile deps + run: mix deps.compile - name: Credo checks run: mix credo --strict --mute-exit-status tests: name: Run tests runs-on: u22-arm-runner - needs: [compile] + needs: [deps] steps: - uses: actions/checkout@v4 @@ -149,7 +97,6 @@ jobs: elixir-version: '1.14.5' - name: Set up Rust uses: dtolnay/rust-toolchain@v1 - if: steps.native-cache.outputs.cache-hit != 'true' || steps.elixir-cache.output.cache-hit != 'true' with: toolchain: stable - name: Cache Mix @@ -157,22 +104,22 @@ jobs: with: path: deps key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - restore-keys: | - ${{ runner.os }}-mix- - - name: Cache Build - uses: actions/cache@v4 - with: - path: | - _build/${{ env.MIX_ENV }} - key: ${{ runner.os }}-build-${{ env.MIX_ENV }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-build-${{ env.MIX_ENV }}- - name: Cache native uses: actions/cache@v4 with: path: | - priv/native - key: ${{ runner.os }}-build-native-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-${{ hashFiles(format('{0}{1}', github.workspace, '/native/**/*')) }} + _build/${{ env.MIX_ENV }}/lib/supavisor/native + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-build-native-${{ hashFiles(format('{0}{1}', github.workspace, '/native/**/Cargo.lock')) }} + restore-keys: | + ${{ runner.os }}-build-native- + - name: Compile deps + run: mix deps.compile + - name: Compile + run: mix compile - name: Set up Postgres run: docker-compose -f ./docker-compose.db.yml up -d - name: Start epmd @@ -183,7 +130,7 @@ jobs: dialyzer: name: Dialyze runs-on: u22-arm-runner - needs: [compile] + needs: [deps] steps: - uses: actions/checkout@v4 @@ -195,7 +142,6 @@ jobs: elixir-version: '1.14.5' - name: Set up Rust uses: dtolnay/rust-toolchain@v1 - if: steps.native-cache.outputs.cache-hit != 'true' || steps.elixir-cache.output.cache-hit != 'true' with: toolchain: stable - name: Cache Mix @@ -203,22 +149,22 @@ jobs: with: path: deps key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - restore-keys: | - ${{ runner.os }}-mix- - - name: Cache Build - uses: actions/cache@v4 - with: - path: | - _build/${{ env.MIX_ENV }} - key: ${{ runner.os }}-build-${{ env.MIX_ENV }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-build-${{ env.MIX_ENV }}- - name: Cache native uses: actions/cache@v4 with: path: | - priv/native - key: ${{ runner.os }}-build-native-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-${{ hashFiles(format('{0}{1}', github.workspace, '/native/**/*')) }} + _build/${{ env.MIX_ENV }}/lib/supavisor/native + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-build-native-${{ hashFiles(format('{0}{1}', github.workspace, '/native/**/Cargo.lock')) }} + restore-keys: | + ${{ runner.os }}-build-native- + - name: Compile deps + run: mix deps.compile + - name: Compile + run: mix compile - name: Retrieve PLT Cache uses: actions/cache@v4 id: plt-cache diff --git a/VERSION b/VERSION index 470abefa..03f7611d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.67 +1.1.68 diff --git a/mix.lock b/mix.lock index 6bc3bbe7..e00e6a6c 100644 --- a/mix.lock +++ b/mix.lock @@ -84,3 +84,4 @@ "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.5", "9dfeee8269b27e958a65b3e235b7e447769f66b5b5925385f5a569269164a210", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b977ba4a01918acbf77045ff88de7f6972c2a009213c515a445c48f224ffce9"}, } + From 179912e1523787e97980769ae383d62855fbd0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 2 Jul 2024 11:51:49 +0200 Subject: [PATCH 02/97] chore: add eflambe to dev dependencies (#373) --- .gitignore | 1 + docs/development/profiling.md | 34 ++++++++++++++++++++++++++++++++++ docs/images/trace-example.png | Bin 0 -> 235345 bytes mix.exs | 3 ++- mix.lock | 2 +- 5 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 docs/development/profiling.md create mode 100644 docs/images/trace-example.png diff --git a/.gitignore b/.gitignore index c396724a..cb558239 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ burrito_out/* supavisor-*.tar.gz priv/native/* +*.bggg diff --git a/docs/development/profiling.md b/docs/development/profiling.md new file mode 100644 index 00000000..2cb63293 --- /dev/null +++ b/docs/development/profiling.md @@ -0,0 +1,34 @@ +Profiling of the Supabase can be done using [eFlambé][eflambe] project. + +Example profiling session looks like: + +- Start application within IEx session (for example by using `make dev`) +- Within given session you can specify which function you want to trace, by + calling `:eflambe.capture({mod, func, arity}, no_of_caputres)`, however it is + useful to have some separate directory to store all traces, for that one can use + quick snippet + + ```elixir + dir = "./tmp/capture-#{DateTime.utc_now()}"; File.mkdir_p!(dir); :eflambe.capture({Supavisor.ClientHandler, :handle_event, 4}, 0, [output_directory: dir]) + ``` + + Which provides separate directory for each tracing session. +- Generated traces can be viewed in [Speedoscope][] for visual navigation. + +![Speedoscope session example](/docs/images/trace-example.png) + +### Problems to be resolved + +- Currently you can monitor only function calls. Sometimes it would be handy to + monitor whole process instead, so it would provide better view into process work. + [Stratus3D/eflambe#47](https://github.com/Stratus3D/eflambe/issues/47) +- Currently if there is less than `no_of_captures` calls, then eFlambé will try + to wait for more calls indefinitely. There is no way to listen only for some + period and then just stop. [Stratus3D/eflambe#48](https://github.com/Stratus3D/eflambe/issues/48) +- You will not see arguments of called functions in traces, which mean that you + want to trace long running processes that have a lot of calls to similarly + named function (like `gen_statem` process) you will need some manual work to + find which clause matched given trace. [Stratus3D/eflambe#46](https://github.com/Stratus3D/eflambe/issues/46) + +[eflambe]: https://github.com/Stratus3D/eflambe +[Speedoscope]: https://www.speedscope.app/ diff --git a/docs/images/trace-example.png b/docs/images/trace-example.png new file mode 100644 index 0000000000000000000000000000000000000000..2abd5dcd8de3bb471ef8dbc1c8081185ab9dadff GIT binary patch literal 235345 zcmcG#WmH^E)AtL6Ai*I(aCZ&vI=Cfh&_M=wclY4#kf6ccVF>Q-?(XjJ&UL@%oNw2C zzC6!fi|L+O!(#95uBw0istEt4D24I~{}U7x6pD;AKp6@O^$7~<0|x>e9_x`BJ-OHAR>F95UU79_Z$A3M?FS7 z)bF4#2=p!zI-T{u$ODWeG$hb}mOo^h_$7vFvTOV;!}=6T-sI4*eo zn$XPdc;3uy>I21>z>;K=TLPsSucHb2g~l&L7N>CZk^B!dQJqK{%Rrn%cegln!q-<< zcVB{71MhARp1F?qH=u+9?GOeO>E;(2Bu7}yFC@Kv4F0OgNPf$Ki&1-fvZ0UUee!tf zqT&>WvVLTHpsBig z1G*6-ORABOa%AYXV|+(M!c$}F;LlVqpOOHnc{Xy8uzfT~g7EwMO7uIEX4+LXVd28B zg24dtucKXeCpMBUVF6Tw(_MW(7zr;)HaXnCjeJ3J@WSfsIcePlEfhp&%r3o4sj zk(tFKQiAvGyAk7XwCav*+@nK(Dht6l!+6;Dg=v6*#LG+FLDSbUtbRp;?aatOvGydu zU|s4M)ftv}*r&*Sly7B%4EwFOKiz*eH=9q2{;a?5KgN}5+WzXegVo4{?q^5G>JPj^+&a~vhc6o!U*Uu=Lcg6Dh$AlqB5_%f!H{Zj z*gixC*fuzU#`p7=n)3U67)Woid+TehW- z5NIatr%CF^F#v-%5*%EG{OR|Xxo#8M(5xzgyKYe)2h9UMiZNY5eZ@Cx}U2;?hE+`O^#z|{YGQ`8vMfT9_ z#($F0mf_4-$v@5~o|0lqDpP8pfse%+#JqBFpnHURq#bUQ@^0~riG_PP={YnY`$I3R8LX2Z*e{$RQc*a?wI=w=C@M9W#_ua-XX(M|c~#0*(yU3YiJiBvjW^+} zIN6}wC&iSmp2mY*psTxR_L+)oNk9_Nzur1lROX&I^+n)ayo z`h;CaFoYjR<`8z!zR3o-1n~T(i%a~H(Zp@9-|~JlI_+t~GfJCj?^(Dw;?B-8I+dNo zd)#A{wD__p6^|feF9RPBzUY2PdhrM8T{Z0Z_*d#jYy?keH_0r}T%jk0Wf!>RC+6Q2 zdgR;4pUPd!GAMY`-C?Ih0E^%Y&gI`@F_NoUjA+@JYndZ5lMf>=_`S+Vf7X^xdIONM2fjc83=0nDYqu(=Dbim4MN&9IUargE%Co2@!GkY?7^`niM z9UgDBFMcmMxEr_#ICYF1Kbouz=E=`;btB?>2P3qTzcA7Ln5yNlJA5-vG{zhD(Y9%A z13k84+cTsiIUyzEI8`F+XzKpap=D^R9rB(XBe@U`^ECsWR(TvB*-CWf4)-?n7W4w= z5ax8=CeQk7NJ8fd%dCO>Z8rTLTONZ?%LHv)CoJC?23Zc*c$|&8sqYGN70$WF*Sv~n zs-5<6hHxH}$7&{O3d{Sn=WSn=v(HfZt69DL!ta!C zhi4Ukby1QElL}hHTd!S&SFu`)ZcWF=)jaF(an4^`lxiv+eRtj*)tJ@P;GIy*N+nBo zXT3SuEh>m^_!m?ybz4iL)ik^iPzTWj->D8$mP5r+8wocFiv=wloU{e=z<;34Q`aZf zQ|A^I6DMZu86SSg)p|OUgGSzAm zXGRHoUhDKU-(8dKv*;l-*)=3BdmQEV_im@GFcYQVI=@>RZo7Z-S$pk- zmm|R>#t~HZR(<-hZ@+tv(=qSN=bC+Rb!_2@dAlRp`0KfsgxA;Wkj_okSL;dsT~n>n z`}`8@2euq95l(+}O_M>wLp<-m&zw{p_hXtcTXFd98kY3U-Y?*MjSU zC+-;c5PtZ5kT=)=n!}T$CEV}r!^LxtbSSr|5m7n1E(R7P9Ky$szcPM_Q-?Z6g;J1# zcB@76%ZB>f1SgUvw4@S)!1hGMDt3|MR(DQ}Ve03>;cf9Nhk=5E!TNvZ|9tEJ zUa^qjpLhQmsVe8_Aag=r~dOWq`3g#ER2K*#y8l{(EppmuaD^8a&3|P zDirAd*B6l%M)pRH=zsf({#i3ED1Q&Yx~2FhYbJ^lyH-g9u|IsRdp3Jn2b^g@;lPTf?k%NrJrm}^P4l#qJeCa zb=JIs=c)k00g}k|Mp-!rDvYSNb*5H5&i}|+^q-6L!{pMwp0xyn;l1P3Y}3)6=hotn z6fHXf%1hr%Y|D>5K6;h{)=9~qv@tFqC$thiq=Fs>8usQRDOA|oN&yDK(J}L?=4s~t zh%JiLl0p{9#J_o#8}mj&z1{parEXDC#JOnk*IWoP4CH>aQ6PKNQkioH4#$Al3)Y9S z*+vV<-TYDmf(0wCMZH(Ip(e1@|3{4VPbg$@jzx0`#Ly9*$V=CP{PHuzFutuK21{^r zVL+LOOwf=v7hnX${ zG69!7wL&`O^J!xa@)uT-2Afs-QuXq9?sYGfx4Xs;88x*e9p_oKsd~$K51PBfqMV_; z7$GfNPdOW&y;LL6R!?{X(v7wRhBO@XYz}B*s^;G<=md;FYv6d_-Vo((A_@e}X7mAE z?DBsXtc-3@VwAEx(x6;ee}#%Gu(nAp!cT1~-_owr1C zmCDfgFKZnZAkt{F3YsodP*zvRDFAHSGBYyP?evXiaMS3zOG<_~9ZpI7{Q1+0C|q@- zP=UaFz9Oqkvz9t;D@wVLYRO8vUu8B>HO5jJs6Ac%d{P@jb!}o|65{gwD-gBjeyw&~ zh~f45zEWeaeqJwup`s~w?Jjx6?r}F|cxN!d8uxD1bv^myXT$0zjBl^z!MF;oW&awY zM6mujdel(P++}PB5oI2=1bREvd{nbp)G+F$C=w#KOrMk0Yrlts%lDMcrLiX4c6kR2YsZ?(Y7mg=eYXe$eO290obY_!A$&8_A49^z zz3QZL0N&`Eno{OHZ@UI{hoE%oPPKcw^4={N?PjdM6z=Tq4#xPt7^U$#nzTE-!LdcY zU4#mMmi8bFEYogOe7o;>A3XQD=kG9Exs6}}{}|&tX4F1g^E^+Y$#x^ZNzKe8^aKN+ zm*yD{lf}>@2Q56VN4fi{PP2XA9`6XB?@k%a&E*@z%XFHj?jFbZ&&yoB_Q$ezrG%g2 zP9UHQhhj+FBwNt4Va5A?^=dPeq^3C9Z%~FNOVw1njzlh*F%F5Pt!nH{b;)ZdPAXq2 zhS>Qi{BHYkKPN@_^-#&#rQ_vl80L{TS)?q3(^9>iyxrqAUfZTwLNc0=PE%9U<-Nuj zvTZauiJ>GuO?vK)zngF2SHU+^a=O>QhuyFD1|bjlI^oUKd>M!)RCWKu*}1;==JkHP zarN=akOk^^@jhxd0N;ovI|yfDW$!91po0&)pClS?hVe^n#?b3ss`W#6~!9I-+zpwQzEmfY1_Onr==;!K4txJ%ynlw zMmRc=|LtLWcgeC6o2~!pDy8FvioQA30fgSZ9nDY4&%Z3pSOeDYgz*P8c|YF`G?+2W zIFAe716--0$2fS)kggx z#KV2Fr5f5bwG4I-o1w!jzK_Em3r(-F%$;Y#?~gIdpcE_q)4J&+tBBhVhB^=*#}sdo zF)nZ&3GsoX>g;je8#m>)-V9d+0W5qtN~HL3!%_I0LzvltPuoWxXU+RNPsf$6!W6^F z6FKkq?{7IGv@ukUdn4iKV@64qW8Q&qDDg!(K1*DP&~Nh{Z_Zt#9S>U(Fc{k5U_|i- zIh?k$mgAin6(wtIzxAQ8vDy zN8RXv*ej2r&Du(n(LU{o-iD5$LOzhuN;7sB^+mw}s z3wqu~g8VXteZln~pPflO4t_g8LgRDBi3ejYOTH#sq@g7X3rizl%yrFO3J#5!aX-^B zn$UUXdjl$4e_t(ys70a9a)UL6cbMeO>#40TLqDNoB-8!+``i3dB|AGi6XH`Cn&y)@1C7P>-cO>m^*%+3gO_^I$87me^(?6*60YBTuBsF%Q{Pv zf^i!wAx8>%U9H+0I7+TA#|iTZmM*aP;_GbkmD4B$9IK25f$DgAW|pRR=ygk@NpLKm zI^z@4_)&=0Q#(}1p9<3$l5$iivW7L})>_7?xE)(T^)d;OkxuN;K#BsY0_C>)uAAB{ zRi|aCpEaICcO{vLgWz@#7wMzS(h>_pOgUeNryu4i8OFz6mS_0xw|T)bvmK?lXRD=o zH)n~q#$e$NR`c>Ai5KPW@Yb}!TFR74!tdc_bo4I0_-z1TDM$dMkc_NK z+aD~s=*0yNl&F>B>$p0qUes55E7-K(sxQxInr2ZACo@^bo7gXPRb!E<^jSBq&iy&v zhz-r7zPhdSZU258t*-eEn3ptt`d@S(GlEEflNf2=Z}HG2#uAWHfdfR3EL$h0LDWIk>!gn#r?>xxfbE(9?{*~n z(P!(QT-BvQPX~E-FF9UUGLN&BK59h@7mpBoHf_{(+YDy&e!Ag%&U?M6m0=&>FikF0 zJW0|qqSl~M$|fu>q`C?1t4;|x-_P-7-p}^5$rm227wg-#-|j=?gK{S!Xsj&q5l(I0 z6uu^%*Kv$$4_%CjnnSxlm?RYZNnMQYeMn*?xeoR&Z%;#>xS6+|ukXL5@+)iYj}dw@ zTQ)uGcRrq5+>Q~v`{8LT&J3&ph4ZU2LljS#9xF-DBeC6IXg040JAN9#g&%+|fZax< z!E{8k&8^z&W>Tu*R9al2ip^?)HDfmO7(QIk3N^xnNT_VOvFe)a?}yY%U>^8*+q!&+3!}SuAk{&ZoIKQx)M9$8aYd0=g_BamXULbm^KN~yg z{;MQ`miUDOeex4bIgG(N{YCOYoK3KUT0GGLO3H5|b0KIO@y_BlAu3e4`yT$QqN=UM zt&!=oDA8iG>Asfu4IHEnliO0;_hh+X8rU%MRNy3MM2fEDik}y?SMni)$B}iUh!wuh z>(PN{|08oKw=*yr;p1 z*N}lf-^B|&E1CE5xQ5}P+|Cv?b_;>RRF-qLG3&p1s0KLfSM2+Ple^r{Oog8{6O0E? ztT}?CMkQ`~-G3re4wP1c@i`r2J{9Sseb2Qm89226cZZc9@)x2Kl`Ko6k|w#;MALBQV&OPBXk%bn?oIUrxf3lEl{YS>oLp}C%yMR)?P1}SCEc~5q@i0B@VG} z|KMUpI{dXq4IvApg*ry1*G1np37~3_lFE5-V#YOvL8tzEM!p)3IoSNDl6iNrh2@z2mC!gK%b}S9IdP^ zpCGdsFd2Gmb58TUj&A1pKcbtg_Zct>$tc%N_Cg}hk@8m|*|{i>Aj`=d-0LRBg3id6=66BGp!MR*E76 z5qdOnojQ{u>f3gcwXL{ph90&ba^7xeGJ3Oo9$+?8Ek^}%Ae8|Og6c$cnojV>!ZRhB=gCSGN~c-r#EUWE3Qbqi5wwn z7$Ms6gL8oO@?HM%gPkVu)lCWeBy9}mClk?@fr_-i`Xg(*=*JRdndn~)(&dkQtIX_? zC|T<+Iks~)zt;4{L$NTF5a!&Dii`Rf8nn-CnT6y8Rxd@lcySAKGmx=**cEC}PMKEi zVyEOA6xodp1H{ya?`r7jDhLRsgxh51(;T0+N#4K6(C%@=4W_tnLiFTAhPGE0FX)8rbH};eR@1 zF<(|O1z7eTR=sc?`EABpXED8}Hes+r?0YJldSjIAyPv?~|g5_y${U zGe&z$;%s#J(!+EC8Nx3d8cwanRB{&ZIF@K+R=P@&5wEm0r?AVzM4nbLV?$}3 z^o2)iMFljiRT-4ZKf9F(c+KA$NQgDob4=ozGSqNv1c??-0$@#p5)}F9=%Kme_%P`b zMn3orI*cA=Ae1PxT&Y}8q-fQQq;fR*;$kV3WZ-$yM^0-t*KuyEo$Cd&RRJ{?O?O1_ zs_#kZPL$2ga^$aJw8)aM_eR;7#Mx>UCIXH=x!-J>?qOTl1iJi7sh5Q>9)<+Zj zMFs$}0l=?!%Ul%sN~94VbCl2{ByNJCT&@Obmd!S$h}}VENg#U(2!C;xdqBeBd%c+s zCiO*V*f_52STk8JST_Iw5$s~~tK3EDQLv=_(Juma45iYMDdlZsVmK!R%9Gw&9l&g zFkT_aurW9L;!#al1H>^Q7Qyp;SaILa;+kd;QVsJGjd!@vQGP}FIz!VRKbdS<1<6nd zGCZ}lPa*JywsGCZGmHA|GRAj7e%{Jy_s2fplcoXLQ@jLsI;HH*?ph9)b$S|pX%DLVD;Qh?w$B1JpGJk52}e8)aG&pNxu;R>cENdeqzs&_=v@^ zm%mn61L-2E2ShN?S&@r_v4^Mr-`{jR4hx*6`$2M0ji^L?fM`F2WBv?|paY~i#V}P= z1AIXyCMM?$3b_CjJ7MxSp0iKCxFP(sCdhCP4gDspS;)>Io__K^%{05(Sp{u7)bhhC z@+W{N_Z|E_)paY`lQ@oE&ET}s&FDvK{k87ftVe^JpU-}by^3x^&3{K%#zC6YtX~1{ zG-qwTFWV%``}oUvAKcYRVFjU%i3L3tj7}x5`-wtrT1EP8a+v?^VEj`Q>B2yG5~1nA zSop3MylM5f`f!;Em|V$}JldON65Y8l4$}r>B{}$y_tFm2PDaOf_yqL24AybqO(KJ& z#0omX+#OT#fc0N%e=~+SD4S8OqkwJIA2jAwwI#4r8$;0a?50^cotl%xZcts&}%Y%w~<9QLlXoQ47T5|5g^X}e9hqffO+-XNUQ^IcgG_oJhW5;c6*jfqFjP z2aoltrg6}eL^1AN;3{7AmXL*j@0_6k-O&}07UnZ(c>dU#?_JP3sQ^t+wvm> znLr<*ElJtXiJyi>?h}7~O4*9nHRAv)?1pkN>w7_=@{~OFI*0fb)sZ|(hXT=|x!Tpx z`&zQ_>TQ60PCefx>KW~!u%CVYU+91oSy(CT+y`+*-|Z;wGOUg0ZkIs`M-1k|#*;qm!JSZjWYRG^cJSg_WcWn8oTk1e^4}fB(L( z=f{DO^yx((KoZpW$I%@D@n(u*-q*Iz7 zbPz&`@`z(v2Z5cEcaH$z&^P#K+-NJg1(E7v|H z^c!YHYEJzMllxUnQf&`0QzU>Yko5|(Zo~iJnnt;>gVB7UotwEq%K(ep%!Bu+5_uD~jvDbx-k( zkZ!kRUbMI*{IKz%^X$AQDrwN<#<{=NNfhr3KdR7!zOP1J#WP?XbQ}5{{`|?5df;KY zNU70#`;A;2szsb@2Lj#Cou5WH;n=0;1u6%=YzA#Nag%;as26n0X%YDl`Mz<^mWhSn zCq#-qy+0a3{CTy$KB_ ze=Af@^dp|b-J~w)<-*H6!w3QF9+GL@guRUN)k`vl@URX@GDugPY#s4a{%Q=9F1 zWMr9~WOR+3VNU&vkjixKiS%S$z-h)tmW{mOSdcz=C1S*uV^%PA2oDU&DqOOqAqM|@ zp0izYjAUZ%k6tz-*OR(+Us?OI>R^@2VSGJ>_AvuP#I8T~?$o#FW=!wyO2l(qXGnl; z+N%6bMP}m?i!$XhR?Bwv%ctf%s8$}ZxH(m9ltF}+X~D1A8-$Ja3t2^oon3pbY}tg( zYb-gghM72`jeO!kMuMfX#J;gqx$|{0W}e%vR2_0caLq5> z?5dpY&>N;i!vqXhfA>^nAz@*SyQ6uTAxQoz$No({-I&9Co>}AR{zPAIZpwHtoyi0f z%wxfkWaiwnq{y{ysaT%%cU@VTxx>@LBkL&~;X1+gjD?rK=2h=~9PK!heZHZcOo6=^ zY(DL>{CU1w={$O>*NW!nK;f41;}CjSUY4nWfi~VkZJx{rpaC(aHt4v4t@-jK&`MXy zt4UfY2RAD-g;9uze@>FRsFImpp|Rq$K-(a@Zg4E#dXJAuuXzwez}eFYH+0>x&P%7D z)|v8f?Q55SOvoo&Bi~xVl33baJ;hkuZe1}D^Z`ybrXu+2#In|SJNcBF{3oJ0y2M}b z5YK0(Okuedb3(Tb|G8rS0U;hmiD4UAt`E5p?0BO{B(R#W@P9P) z+zS-paT$N0yN6A~^>3+)7W#Jv!Vg2}k_`oDhIioQN423uds%fSTlkAj8N3MF!^7(q z;p2D&()T+1yPsU0XN2N=V_C-<-%Z<%`lGpWdHME_{H;b^GCf!FeX;^wUjAO95JhM( zpDb)NxUpmk^pEq^G`<|6%}cvH6Pv7Tic?MtP{`-5S@ez2Pp2R!?6yw{gt>jT-#=;c zWL%uL;@q{6zsW5d!GE+k6KH-0zYlfjJ&deqPxi0q3)^QlL-_KE38DBJMDpolWT=tj z*bs_PI%fv_O-KIBW5v^DS0&~e#bvWX3n5AMt-Ls=6~H#?d(#X4M~#fEvZT z^|I9)QI4gYx?RLB@T;4rgcML5I*>^#6n@jsf;FSzn^VE?uN^Q|trQz!zXvhBt7l_5 zF$g|WRzj-2s3`LGVw&p)E_9K)@OAddQ){1S zFWf2eYzBo1^5RUU&`vEF=!WPAXhahWE~!UYKbMqN)O0eU@oYd@%Jtku`+dA6zu`<* zK)J%EoMmlIg?QKx)o6yST+5{2#~9R2vBhZ8wBz*W#`!+xy+{`*G|-zhUTVMSLZW~*xO?=#L-yMg4kd+c<>MJ6k((sTwx<7g_(Ka5P7zBh^>;R` zk$v$80|UcO?5oGH=rih9ms4Z{Ri~vgiL0~7f;lO^;R|jH@BFAxr%!~?`?hkegXrG8 zBqu)~=(cMKKR4!_f^%41Y^?NPrKrrvU1+S*^CY8cTr^Ks!bg13AL9Mo9t4fxO?>F_ zY5awnTwdR%xLovKDi>p{Gf6|nNK&mIENclN*i%FdP{qxn(=<`hD(us4qfIY2-@wn6 zy_T3!*1pQun7|&huH+#0K%w1;Jxr6sk{r|V!vE~_=*rpS#o(BsvDGIrDp2B&f{A1|iK-7jD3 zYpkrri&KPZoac#XLfyOA#|3K7#)vxp3h^iU9b0>uh&yn?Gb3 zoLp2^XJ~jo9To*cRf+y4E+XjkjugHqoxp8UgUv=|SdR6iSuinri7v9OUh+Gft5C@P zoDr(ii;614kJYplK%B-_ZOUKF@t~xwL4MCnmxL06JA-b|hs|V9+4h+m7n9*P#XSFa z;>>m6&6sguOv+?qbsta1tJ!gVB)zSzU~VOy=v{9nV?vx$FMNEXaUr4SUbXVX={NqW z-8~bzBC!FVMH9AVbbn-ChW>mrW^PT-^;i86uX?xU6K<1yklIM1@7g0b-Pp3WEF7T; zLJi2CWA}I>vOT&-Qz7xdGcw%g^7glG-ki($F>vS#o0`d9gjvuWcsMDfq;AbzE!XHd zyLWmu*>%fiE>F@Z(|~VycwQ)9DV}Y-Kk$2XXXo`<3ZdZ)syi@w-lQ_Vv53Vb1HbNM zzUUisc2!FGwtwT^;8wx$@hrE+4_Dq-KWLX{5@4qJU|+>I(`$$>+?HIj8krYnQ-ixb zXEco>P_0~7!A=4E1v!E>>P)vkB7Y-+&wUAb4ane4!AX9-G$g=Rk_s2zb3=}OOzMD` zd8W<3H&p4n0!YM-z3g!d_=M+c+yHfSUcJp>W!w1Jh z3t4+gj>l-HVpH2%N@;8oYd+7`^Vo6-uaKS+n}GH?xKQV4+6t&N9xRN^?=Q$X(K{p6 zu-{OyQt`S)x|VY7@c?3 ze&iWsVzNY$(U|Jubn`8 z0asU%H@$Iw?vkuUM-fVUyLzsol(gb@GyG>w-8lPrk>XicIR|pF2z%<856^Cj0+14> zP0D`-M3^F^(k-R^7=Khk2WKq96qn_OKo5Rmyk}M3`Q>C1yo>5>x_UW$QYK?e@dDlT zo!m&C+B3YYS%1-hRcZGmDoHD}ZDaA8Ki!?%m8)b1mieMxTGnLB< z*{-RX>e@V#0%3kI=i0&e2cHnE^HUq#Uiuf6+|fb(oQ~s))}srX)O!>py`#qcO8$z3 zjwp9MCq5pyuUu^08z&EEsUa);714bracW^1n0f=G>=B z-wpOewdl{=->c={ow5np?b{+3U4x3t`apb)@r#ao!+cowEw1Op4inJuvND$f?l0NG zL!P!w$u(T7`NvHO#-np1S`Hv`L6k(9!~=dOeO%ttztGOnd;W2)6s^`Ax#vkmr!1$V zzf%6Z9B)goT@PnzTHdd>rewAO*9SIB6-+jws2#+7#eGJXMXz4Y%doHBisUGam38sB zTPg{KOHu@1_GpB-(1xErIQkEcc+Tc84W9NFGAAxx&jZpO0YEN+%S|!f?O*q8gi-Bh((3yq=|6SP4+3a_)XHS*%V$G0+zy5MBh}ETEg(vc08QqI;@_zA9 zZa3B2L*fDuH)uBE{A@PK=l-_(7zZwF!HhZZ9HW{a^y}mg2TRsl09WLO^R@eR((Eq2 z;Z1cEMNQNiI99(H4SCjdg2>0lhRWuth{YDE*;!ezV?|OJUv!AU*C8Iexz^H22g4Cx z(Ij)_NqJ&7f_mie(}XdI|8&Zj>CI$LE3?7~qT+xz1iIj%SR}y|NGxxeENGs3T><~J zG11(TV+*Xk+6;UkRr6XAIeYRGhx`5jkNibHi(p}l_gd^IEPDgm09V(JCF9JkapwWj z6uI!WQ_yS7yz#_i`spZ02k-N}XrK>~*C2cAXwaA(-V!I-SU#50rB#Pae9$EM{a# zjxVS7d}>g*>SH(`R(iYe$IHM(f)233dR+5n+$alF`T0y6a= zied1{lrYGul?U?)9vA6+U*^*;dQ`o^__&J_H1p7o2ASP_9NnU^g#qx zz9asP@YcH9Hp((8OCku`?zQ_e^7$;=^Q^k1XZXT6MORV5$w+4l{LBvTIpt9D^M%Z4S78bo!R zKr93P=t78w9j8cUif^s6s? z{;vf5rm>&zk4h_-jzt=#|4FaK<_r;{L9hQr>EqZkmX?FuFq=iiG~v-=;fhf+xswK1 zAwU2u*sLEyW&C>nX)#le<9jBR;SUYx;~$CHBY$!zClfHjLgf>A@NT(VvZR2PS@mB0 z3;-@=3VFBGM?8a^GdpwRtY3XvoQLA$_|z`LQ<**8e(1s;2ig6!&hdm^R0{`#6(2iE zU1)My9zqY%mWd|Q9#Qr-bY@(IBb-=yp!*+p9Q%9-7j*QI_Mpi6o{OZ^a?od%K_t}p|2!)gLThia6){t6==jG@}xpbaQqWZ?{ zv-L2>|2sBAC=qegF$NLahrJWac|2)$-&egZjQ>F z4pY}9TT)jnAC0NSs|V|rihVr~Im{Q-+SmSD0ARM;bZBX@1^(VZu%-yy7dT znc7`q3x}{-i;?m1q*5h3BUdW1oA&e3<{HCU>gKnyK)>!Gpk6wYc7)t+x{4*ukM*=h zNj)w~O^|+v;?*gn)y_Gr$oFxak#5mOgf-rVcx0dOt4@duOI3&O;|z_m#SjNTQP?Xr z@@VyY2{PS%h*VLCxFHq&n;Oj|cI5D7 z@B{}dAJlC>#Z&feyJf-q-#D&3mLVPM3ehxodyo?l$(<|cX-Y3@*(SNX+nnuByt+oylTg)nu8gOpr=No|@x8J>p zgRZJuqOw7sdVIg4x`fc42&l&WpqmWr7gg?+q17t6)qhm=ZH0#l8hpGy#`Bhua1j@fhXWe^=bt0Q_6ksZV`;Xy@PJw9AG59*u zU@zIkGCKdfwViD7z#Jywl2w?BQ>`KWrAi1H_=wdq+(mVq|I3v7kb6($ki*T_u{1y9 zx|Hi4jz!5_@_s0P2ow~0kdZd4KEosO#N2icd)&2sYB&yFNDX?f{ZJRi)nz|P3}6*riM)D4KkUJy+x8v*I)f1! z-LhFvhd~JRXeVLUMun`89BD4=eH%8bj zh?kc3xie|F#t;`2`kpC-`ISz-iG?U!0B3UE*I?f$Ojy#Q}eS=a{pP+ha z-(OUNd}K-QeDL+(d0L%1tTNRTUkwhIj+H(n0MvLS3~Kyl9~Ch#FbM4&e$ z=?^RPF@kqOR2~!;1EKZ+7yjptbDzHNL!v?^Z_EOJZyT zD$j=BK)%r3H$nl-xWb7Xw^K2~(5A?ZFuw#<@gU3n@#P@ntlQ)FjX<%1PUTe;#5g)+CWBe30jPFs@!z+>AXKQU;Ag3iI{lq=N9!1vZza zN30a0R4}^E-;c(-uBogV8*FnSr!kGAB-#kn=!M5Koev_C_Ip@OEa;F#O_(smWj|Ut zQf@?HQ?*-1IhECiH9EF{exbNJyE)on26KwVK;l9w*Wg;7CVq^YPx_BfC{SSvc z4}2zCUev5Xe|}BWKJ@rUO2#}}j@RD;sJ0slctL-$Ry58YgRUw3mAm(7RvzsM6{Sbu zw?+W5h(Y#o|A(^ojB4ub*1Z)`KHo=UWD2C0S&xHRpZLd0oHBi>pW@9~fY5+|EVd zCUd+p+?U)&sfagV&Hy;?{@JSeSq^8~5`vV(iI-nK)2Cjs49w=0T>lO!GOKuNnK!q| zpI&OfqjQPSOPT)uTqth;FPT_6sQo5eX7UG%&^*NUboP1;1`DKrApMZy`v9B}uS}JO zKgfycc+!T)Xy%RE!6?R)t`MzIfqo<1u1{+O!jZpC&U_{Wj+?gO)E1#!T{}OIs1@6Fhz)t$|>V;w%&|e^RFzz7<21?<)u%+2RW^@t^ql zI;lYbd|(}@5@2$d6WoalSFX&I157+g?626TJNOdNfrp8z{0(d5*)KC7H>&FPK$EA^ zo17zZobg+`frU)Cue#z7LcE`1W4}>24VV8o=S6yKo+vdAaH)C-8Mckvs9=<57B}5~ zls-<4oe$_6UI4Y|V^tzP4T*@v68o0>q&k-siXT#NG$f)Nc#-(7Y_cgtK65={gWtQKR)Q>E7g3A%N~`J@BaeF{7kmOl{j+ZQu(@gKo_A=-vBze8I4B_~s z5y_bQb#p7U$cpLM&FbLJ#o>?Tyze3qv0&VvB}D-jVqC%~;#7b@@SiMzwZ&@7l{64A$+}8>S_nU~uCn372-Rx+WV+w<6=ri&*3y_P!@ViqkBIChbNg zZr|DD;Jkv}?=qp!43OL|N@0!v5Fa)Wm~&?)QlWq-Iq{O@rRvG)MQ{xp3Y<8-huN%q zz!?A7iSFcz{prawrgMB&Zesoo>Ra=8N&2t1-v300y-8c@`3wd5WKin;qlwhKPJ#JR z0lye)dS7a3z2-+w?&)~i(5(kte{D9q`y z5=n0vihjw=yUU`@8~=iSaP|R9y1C!q?dn^qv_d?IU01-;c23J8)2>R(?TIn`8gR!c z(_3Tx3d+WrSoPVWlbz^hLA83-xFYvvh{0`9=j7M4ipE`w1YGe78H!C(Q;ZjEa}9FQ z&-J!+nrmHE+)($4bpk(+_`p==5Y^F`u+w)oR*ueu1^miZ8`=c~UUnz6?UFr`==!6r zLihf6d<7LjS}w!PY}_5PJH!8Bvm`3m!OXnUh?xKjp|4F1(=6Gi?SB6EXAaOC<27Td>)`#sd0*de7 z;+J|=KHP#Uj`6omBYJnPywqB8af#X9T&(|Fp+7TyFtf$@CcZfWc~?%s;zEW-j#}3t z>z6||+XXSg0ox?3`3-(QQ_{OZ>oK1Fhf~WG0lrRlx1SuCH=ypDoOCIN-knZnUWka+ zsbo9e`4Y+ae)DmqQW7EWyVj0xJDFHEc2E?msoSGDQ{P~~ z=pXpAR7@RnL?}Oy{_3PhKJU~z8afy%c_AtGBH73S;c1M!_h*0hca0*)GCwo4yu;|w zmvE2sJgvsjlFf_8I8!zFPfQq@Z{}2?%u`a0sk{aADpcSfU+Jz#l+z!bY1p%|Kxc(m3I$ygd_A)gYWEkz7%d?rxD6 zE%|UG$N139oA?J%kUZ!yz6U#766VXSJ?sCOR-)pO$OrP>YCIA-_ka1Vxz4R@otHyT z$G{NwawhW~gFv(}BGoaPJM40}0NS6S99XZO2V&@pLY zqqkKKSojslojC1RwY$q)?}qMwtJ^3~(wF}^giFtxvV}lo1)d6Cvicu7MyN4pImvkE zuu#R|_EiCCVe(_#(b_IDx<)mYfjN={;gsEobfsVbCCW~XFw?VR#2zwvV?oDb7ph9& za?o8T3U#23Mnh%#Msw~6fav0`Eobz*bNIqNGTsjV& zTCOqPD$%c~<@X6>GT?mvfUJ_N^*SzG2Qb&K-B{ttV%t8R(a-#Sc3c~WKr7Yi7ch`{ zM#X;D)|OoVq*fG=lND9I_F%vMF}TQ+LXJCgqm~WWvOu*&`*xL`Wml&Gkt3_BK6|>&WsE# z?Ikl>Vg^MIi*7C2C=ljnr==kOXk zo$TE7BZ=um$EoP+s;6m-pV4Gh*w}`HJOzCnzzn=(u@V#|Rlpz4Jij8cS}h6pnsWpM z3-!IogYNVTn$1iDlrSE!1wXC1N5S4^^C(_~){5hGfaNP1Z~Yf$tBjxT{u_5=9H|GBP$A2IFQHQj%0e8+wSW6x1<~;rQJo6 z?N~iU!?p%;?il*@y^G0}%0knaZoB{C8#rB+Q^fjzuM)yUEKoUN3$?0C_zbJVUTe&H z&-`*FU0%F@Hbn{UlE>9@!G!}AsNbe%HbJo#&^EHKm!;K%cN8?wZXDuq^Z7ulQCoi1 zM0Mv3=8BW+R?2T|=FQZ6zLA|KuzR*T7|KRCQ^yi?(P5x{3$G^xXg6+j<)kqCQRGk@ z?-G^^Uo8T*k6NP5SPC&d*BqS_EtMdNv{-ftmE;MW$UZ3KV%9aY7K_ZUhN5R?Sd3RQ z!rcD=LZMY#UT^mII0ORM(V-0PanbiIPcOdg{___1F^)7De)4B})eqL@iUs_AT1fYv zd$?6~c6G!@`{Tq(*xH7)%JbeD#6b0Hx+2GHdaF6H%m6K6mvFYQA$AEIXzLR%@jVm8 zcGYPcZK8u!)Qt${hA~kSa~@G?6;Ob<(QwWy15?2#Z%!F1h^y}D1#nTqin|h|ET(XUidVM()#QlY#d@m+A;SXxz-_0+9whD3e9D-}xxIsBwCdtYsB zs7{qxYvHV3!I!4g7{D%jUo!%%bfP6b|CPJ*OGsy`)4d|XP^Qwh*isR7VXHSPD&^fL zLnM@a&IkD~v!njVZ>tzYMz&M8=VWA;6uu4Nj~uFN6E=r_5F{L{W4n`*n%i|4B2I#r*g zsd;JB{ou}hg1S5t%an6WefSH(Jd@fzwq5CN9cU=tdLB0A4Yb&2{3>Lvz#W@{R@?Vd z7<`;M);sSriaR9+IJ)RJ?KY}@oAl4Lj)YfS4;u3^1=RBDrs|wruEq!t;aa*{oBevE z>bLmj4Oc7VXw|Q~fDCdO_*eR|wGy>l{MH46W?*e~Bt4RT!Y6x$45E`Akb1%(F)x!N zq@_$qp&poj<%V6h(zH&EEt0M>M{;K#jfqfv38y-IZ&LNQQ!+$4mEs0w!q}n@x@qu8{KA$hKEDBEJjmh;$095nT}>3Tf4s* zYb`b#U+0w@car#D&y9ZtL$Y3hpQmTe)%BKLUnxVgYM!Jzsm9MZw_bv+)05a#RcguF zgAV2!t%S%*e3m_keUdsBpC@k7BCNVwK?k(Q$pd&KA;(lqRf=4u%g80L$|@m z%8Sp6o31iSOg>3oQV-cdcyk21W|gsUs*`xCzseks0?5mx-!^=Ps_+!g{j)FQ-9Em# z-F+#r!dU;%(C=D))#_E5s!UuqvZiDC^R*NoAjH#hNbezE($Rr8D*)xPV3-j z%(oVOR^z-DF24}gN)MA{(`ge+Fq=o_fdf|yp9^r0 z)|%;*+HR}zijxdTX^W7)X}`sZzY+B(hS_z&xfQTAmD9viMKOs`Gzp`^Vg?gr7oLsV z5l1oC;ImsC-dM@^ov)Vg71z9qfPelZRE(vQy!3cXgFns~ubk%0-Rz)#m6(O@)>DPs z2lfYMQI&)F%cNezlAFCKC`vZ_%Y?jGs;AoV^ma`w8}V?kL~3IABTi6G%Dk+~#UKM< zJmwoC1cbQv@9(+B-ldORBnxQEy%TOl1_6Gem8cp9YFFr>-WOG4y4h@i9pRYb$R{jK zNBy98GS*4SY2C1+<%7$$Fr^fdUto7iHv5`{y>5%) zkWb#3y@smPn?cT^zD$GlCx9JyC|?S<4p183FO})qhY!A1SO1tp{X>m z4jt~qCT8_67W1o?6F)Bgc^&cs_x*n+s7JgTwL9%=S9f z1dJ*c-d&kbu@O*_vs+`7&Gw3F>_%$W$uCzgL~hbi?P3P5!GtJi#xnI&hHtp3?EUo> zVGwnQPaa3qwN?LIHMrYP=|q%WD}`b9P`s?DKwNR&>uv!*q+?i%R1K)a#pYK{a5UXN z%Z119ZSUsrYyt|IUhi__Up9cLXny|tuBywReRld2c{!^v`uZTe&JJN~(_?YYfkWi; z_eD}IZ;z@=ipI8dvSw0JQym*Lwt8xlJ$(Il?*Dkql^O~=OZwvPDNaBbm#2u;8cCGD zhGfa0vIw;8;~no?|81ok*>NY@8-CU3g(N<9U=>yh_> zQ3-@ov>WUEs!Ea*`&)mXZZZmIIo-i-{?0JlEk!mrq?bw_(@&1_EM+|*BIN$Ny z1kepAonoB#bBaLzPQ2%C>!3_gZ_iRzSJb}My;Wgidh>xxzkZJyQ{y*?3_g?lYt>e$ z+Bt;3dpWO3EsHMbQ30YG6yqYF^312uewXCi&}U^fjM%-K?2^NR(RQ#XK>XM=W4R<8 z6(85^RXSN;cYu>!u7$CoQe>1c#Loo^66Qu)j#8t+hifr*S z%LAr`9u-u?e0vH3T)lw?zmpsRTjYp&ZN1O6&}bgoA;r9(r> zMa>|m6O+iWQy_M{WrqUd8tY1| zAi;JcjR$%YX7tgyynk+JQ4q0XUHX^|y2(~^Gr&2s6qQ05pQIyqZe6MN7ye_3;&(N! zH}0uud*`ozL#8YvzAdk3-aM?6FK1Fo7v6UxI>f5tYj05uhpbn<2Z&pjG@KDS%CTu-$eS{;VTqVHrUV@m*BJ9atqjh1J%Og@7TZ`bdq+`G?Bnjg|1jEX2%O%@SuCD*hDqkkfyzG=p`fYxa%KI^vH~t<SklD>)#A(51vY`K3H`@^KsgW!!t5XEh#;>TT_sk1~cua*=#=R^7B z)TiLXQgU}=v^L&`to5*BbkhWbt2i{7r-ko@sUK!V*sy!S0LN9`lls_i^5)$+nj$Pw zLWWxS#!EOLJxXw2Jf~V!+bT zofp^9BZMf^H&(#Xm{r*ftaIlcVbrwA4=QOI_h(TEI9PlSy~?@KF&Uo^1(bm?3ywhf z-%0G;}490;(trnst` zNFg*#XLP!c<)!@VJ$4uRw(w&M-V1UwuX7mMmi`b^pR9du3RVA>W}Eyld1#7R(BD4Q zIOxK^ddNDQ@o!1oyjQoySlP67IG;eDlvZ->*JpI=F!2^%2k}pT+smqgJoaZNW-FvU z1d3iz{7pu3Kb}X$3Qo6Xj)a71+%w*|2RU*+ln4~QR&IM0LMVV2d>|2=-ARA{^d1Kd zfSSP+<@sQ_H(t$4Q1)=Ci_m#hj>I9QC737!dN{@+2n=~zsmD}QJVn&4v~tGa{6Erk zh~$3>a=b2ZR`M|oyuNf8o+rPI23;LXfXgMou`zG-oA+I#p(gsxtfJ2za4cx{rlsol zi79!|N>$ewAJsT~HA=YD`nx!!YuZx;Hnl#=?JFjyAp4+pn;|Zx3sWF@1ltYb7M6Q! zE6lo){qBzF1gJnk(R)T{Yt08jxX@Zc4%yy|+l&_uE$)4*^tK7;M|`<-9*mQVMwK-< ztW6|PPL^E#BJl6qk_-3pPbAE_Kk7ZkqGS!e9)4;bNyl{Y$9kENv_*Jqy{E>3Q$HW= zk`y54%$eEn1-VA5`mB^?mJ^H*eh3(4%y zU4oY`0S;3iK*4x{IKh&g;9B5MnCP;;5K}ebIe489{@`#-zSQJ(OBqbf!^KT=7`g7P8YaPNE!kkh=YTLKh-@wiT$le&l-mcl;yOMfc z5*GeiWsQJUb1t`aoRj;E5}cjdj|15D-|bc=AZ>lIN%Hyapp3+GGF56Jr^a5i#rpIl zwWAwDrMQq_^eVCPqCv=k#hh&cLv|5G&gV+sl_*8Zv3DJ7a+qMq7=e`Hn-Kvw0h_l^ zgDPuQ1U|3@^W$pT=rXlDV2i%6(KE!OQj1Q+MsXjg)l8R-$_iLQIkM;hXi8G)H*-pr;1&OEf4f*fV1v#=@WE;x2DAqJp+pL}NE>{;dC@bDM zRxxDD1xq#Lx0Bu0z{gnI$Jw@WehzU@Tcz$-J^G+ zTln_nv-;r^JPUdo?OPpn()U<}+HDKT+A`->Xq%2ZgKpt)Z^FdC5uw}g zC%({r04&POCAl~A3qTndS<38_c0GbY-{GPown%24Z<(=npf~&7Ob2Iz?PYE|hX9+X zdO2K_mNFk8wd)ntENV`B`Oi%%9hF=87G@#zx6hKgr)GX=zSYWRF6|&0Zj0emPcse|xMUHwQ8{Ris}RuM(s@CM>kz z)Cv0`&?;=bzH@q1SG*_|%rij96f+35PovI>T~y{?sHz4sOQxPHP3~opT?G{Lb8TZo zMJum|%lsO4&PtgB)^>m2Fb?iP?-(yNkW<%V)Y8Jp4VWp55%xusr4}R3KvJR!4nFGp5Ro z=apoH@K^kX<(E8aPH-*Qy&-gQ9O~vZ@Cd$_|K_C5aWlkhE|dvZDe76i>a!>KoQanWxIh;-3J zZ-t6i;vz^v@ugv^v8Y`(`0`F8!>-aingod6xCsTBgUsgf01xLDImwzB)c&ccLw2Hd#6# zMv=`5xj3m#acyq);NE*T)>9$G)ynfVDBsw3|6FxF4=&HLKl@NkQPD_=SX3(u@1;sY zOVRhuE~$MEjFmnJlNX2606ky@-T20?ju#lgZPvAO&< zWJ~oFL&(IyzFSQlq>y~yHJzjdo$9JoE1MmX#vW{j;|ik zf5xgQS~!YW8~r5t@Cu166kK}fI<6%?u`Ndu2AJmul5+}ePP$okZj?Rzo60$DEFtLz zjEpkMmIVBK2W$_L5dcKjA!ZeQ_q?WxJ&9_#xt2!A#oB0FmRZNoc5`65?0l?DC(nan z++_AqZOTvG&XXrRX|X%xGnKUl8(O06_o3X$HZ3(A;<8Hj2d}ZU+sO_eJ=MkA+1!NI ze{Vai4e{5iz0NJxKdI@M!HcA!*JmEPXR`*(f&!kjxuL`+7JD{{cnO%OM{1NHbg2vG zYD{(ZvPZoc&sT)?$>}H)ZlpwxX}~#PZsC)#VgcGxQvw^QuXqpj0vgejvdxiBpyQ_%~FyXq(4Y73B3vb7~sHfI<# zl}8^G00vw+Q>}I*d16XjeiS^ro>pf3hhHHDIpTIrzh+weQN0lI6330F=magPHw`^K z5g6v9Rq7SHN$nzr;RHDfSGgG^a}uVr|8t$hdk(3HKd{9X|2c&kqV`8Pe4BmLy%}_+ zF<$heROFI0_L32jWbC{`fz#(-_|*XNLI=vNk`Xgj=WI5o)3c+zvqF9HfpD(xIV(n~ z(o8%7)=X}gMN>yQyU~1O&fR_2B}UH*G>Syt?9Y$iP)0}Y;7RA?8oLmOw4Wj1@Z^l# zP;z%}T$46K@&bO;Lw6zY9g;AI&iUaR6}fwPg3M7A$D27EjOxEpRxyKcrtd-s+;FEO zE5NDCFokf}){9f_PkMxXb^0hiueo%yC+U%SeglUDKecQpW7Q13q&ha90~Pg8Y)cn& zH;vX#P%vTDb^|XISJ3nJ3ugf27@V^ag9r^=Pu1@Sh-;S%X`eRtj7c2`-bIq4M{R1m zK#>XVzUSIcsl^;I9prOQ`l_+GHbfj4OS>CurK1_X&=xt(3{f1Kb1Nn%Y7sq@o|Zb0 z0lC+wPC=3wf_-aL(i)V#{JsTyR(#a{i{JZDYfN{Hu2p+7J9|sRTyErh6=sY$C!IrR6LX_+I(dFym50jB@*vL^qM_tU2b@(~}tRMji;Ngp@+9+zq{ z_@ku}%~p4~B2VwflPND$`l|F$)Dx;v+rcJONmoKbaHKbpfFJLVBe4mlImV|;r!{0D zX)D_~3JPcZ^Rh8Rx#F44#8KNyT=fEb4_MRQDNnx=F}PV@x`I1tQyL4pGWaw|O;rXp zm>Vi%%-))-^9u%FqM!Bk3geoL)zQvWPbkL@P4h)ENIzF$@cB+Scd=_*P?`|%``wSB zJt=FoAc}D7)WKZY*S0}NP%LVOq^N+svnUf`PFEu0EWXasty_uuQvK6B$jGib$jhd{ zIg;3Mt(rnbeUf-(pU12;c^g!?U+ehwOmVKvA(+0-%43!-GiJiiR)nR!1I26QpX6ZyGvlgTloA2+}8r;ZFt zR$qQzmR*lp{+&|$i{_?Xoo^DoWBntr_g?WX`mDMv(`gynY@WgU7Rh`(3T zkWtL~8J=09=dKoR+#c`8Nix$#mhvH0aEY!t02H6*^fr9?&gT5pra)? z1bjzg5-9KsNmc3e$RW#`;&H?~`D{wCNBwPC#BSDgipiNKTqSi>>gny0y5n*hw#~nl z#RliNvj^)9E!5f}_vfC&_GvZ@UvfAQNd^w}xEVsfC{vbmB^3-cGqd*`t>2ATJC=}U zk@$MZipOY7BD9tVfB?l}_%0v;mkGl_o$(d(%u0R89U`V5pFHEK+S8S;6&We3*l3f^ z5|xOftzR*_QO45((xXaZ-uWqIl=lh+WabPo`_DleF5`0sVWfbyBYNxUud4>Bd2(`5 zOq_Web4hzP=>3>wiq`E*>Ge-~;5T;%$)As-^@BX*{S?u-HwA3&ni5X6uMyaM+dt7Wx1FoEH5WqI~0fKb6xvdHCoeZtADHF zS$P6gWdPvdI7-ew4| zM0`KyJF| zGOUK|a~x%FK4YkTB~S?HUx=#D_aldh7AGfROMR`spBZUIz#WCa`3Ke4^@3w|Z7+U& z*G{2Lc`C=rsg%b5_VIYJcDms>WhNn1a;(D6PU)8r)s)PKDE1d`nzio3^0|4&@3D`K z_}Ks-V;!TvkI%_U9sheqDn&QOF(v4z&$kyt<;k$gE7lb9&x0(OypV-aQ ztCXnNLqPJlI;t_a|J%r?reFjjgS+}}fAedCu85Y?(AVFU%KkE2H?$&qt*8zb? zDfzv?6o!X&F+jI1@S7p`${oVuU}fB=Rr%@}k;J2#w0g zs7PA$otakQ<%?Im%1=)!@N3l;AFIR(RyF(X)5i5=e6M&3M@Jiw>+upqKVh*{t-42N zsrV58B_P8nc=)WyxIg^5bSQT2fg7%YTlH zS>eO2+c>!Rr2pHC;J0^0F!2=u*w5qo(4*9$c9{yFZl0$yY;8*%fuxSuIb#`j=jJZQ z6`BvL9E`CKc{2?`I{%qo!L@m)eS4HBcs4=x&nW{I`d$qJq6hb z0mdiHZ(Lmk9r#T(MW9(K0VAlHyS_}DJz8@JP(Nz?%NbP~o7Y>o*Q+jJOqJG5n-KJQ zS7DLib9?oUh`#JFE%9B^dB^%L(2(#jCjXF$Gk4bEvq|gdOu(vb%?Umsqch-?_!@+l z0E=T{rP6u#kiW|QiCEt$KOVh{g1ebaw5|*kHEoq(B9$~^Ug#Wju^eO->Vj>VG!K0j zl(6v=FOtH*gzWGa=*dWn>UKa;WdmJtnjL(`Bn9 z&K9Q#RI7T$?=9eb(UdJc+nfyU%JrMUb!WQWzIm4|4?0>QYZ}Wt4t{o)3w{Hpy&0%tbPI&GQ*gIT$o=eDqvyjF~+33z>>! zTC6nmS$3_v3iR{Iv3G{SePN5AGRRi>s7!q<#!DXRl-G`NNnG`EnI!a{st=hD2)n4FPRC0QS2+1l=i zR{`^dox=v+LcQ*xBrA4HAT4pU-Y`@67FiRR@@(3#=5Te_iP)^!4T4^eFsA)|$&30u zx+y9HjLf^-*2*6^yhM5pre9wnuALa)YV^d-qUtX!BgcexmQ~;;wWa_Tf1Rn5vPC%P z>@C)RsHN@%`r_$y!PLCRzNdJ^K#;4afp@28OHa3!pWVB8-ub`&*v7@n7LsiIIj4r# zh)+jzqWNmo-L@?~utlpX&Y#{m=cewv6BZYdGP1_5sOp1w~rm@P&Yfz-$24-(dEla z(tGrB*Km1)>lp|0`1DQGS&f5pP$){2o?XllUCmwF?j@TRB49e` zruN7l(*&)cagpF2y%?Bq3bE-*rh^rmb--$zRveOodxcIfCrCzkQ~vr`KEaVZE-3aO zNXjFp^NB)pVs;rc39G?^M&`L_jFnrcn8)xSZV~Z&{Tu8X&aBdFyzd@7c-I z)h_z&Ze%Y5RL1u|H%n~`Cj#$O4sB$I;W5_h)vxB(l1zd9G^U+UWgVt-d!$}7*2k6O zZ(4+(=R=;MkG%OvbTQ5I~0S{NnF0}gtIOaw!J}qpj~CQSa#o@CD3wW ziQ~%U5sIPpORU)gq$qW3`4y((y7Anj6#bh<{*m2b_R{np`fSK5$E0oL2J&;#mH4ll zfi&4VWwi9pXiU*Lku#J05Gat1A$z=TLF@`=by5R+TOjbV+l-vhbaF?yzQVqOq}8+- zlDGeVWRCyJB?^u>v=EqnKuKBceY5j^{j8ejT~Dlz9l_0UL7Q&)pz*`7d+Vf%!HhNL~V@vkY(qCB?|t zTx*vwzOl#Xv!OF!CYDZAT>IsK|Kf38-0;V-puyJvGo{_ep?jdYLuN1~ora2Up@7RG zNCHc|!Cn;=ww>z{i-O%$CT_myoozS~d`A@1W19U$9FZIC*(LY~$RUGcl{8as0I^tH zSO)mA&Lu<6i{ro*iu`Gj-JUrlHO8meubEH`)3dRJ{^Efgj6l}!qNS^RzAgHkO#pL9 zT4nNI=F zvQ=4zzyw?x4{L|jD8j6kTaj_jA|nYy&_X^uMj!Lz+mU!o0WR+I0rQ~A!&u?rciJwZ zJ{#NZnq64t1S8iy%I@!vDABWYPm;YUxgF9ni3{P|Z9jOVVaK_R` zWm1mJO8Z1tM5D9fwYHGnh)d5N_%RHA4inZ1y1rGe^L?VRfsAi$ zNN}6$pWP`FrrLUhb2t;ce(sar{+TT>aO}M7%a{@Nbl1rkZCN{WHtEc~Djr>8KcDMq zX>_{7U`Ds=?>q>6{7^|t(OITfU1s~rCY-}BUZtXcaIsCcEc5IE6NBDEiEF1V zJ9n8WZNmpZEfx9MlW=OjhQpu^PPxOgN40O~7m!5aP}hB6(9jU4=E?O|b3K{!f;+u9 zl%S_-m2TjXzH!wSi{0!suN*Q?F7TLi>QldPV1FM0XU?BBZESlnyjNztK;LJK7k+E6 zv2$a&kEF?}es{r%%4-{l4DtR4^pj=~EuX2N92#wGe?sT$sYSoU&ihKk>j&+$J$IPD zHFSv|`h0kXt-J2y8JBS`o9nux2oPwe0`cUyKNNNHLHNU^n=um=PFc~B`PA~?#R99byQZkd zZ@xZg*E)j#eO(Sj*Ef{ZPSf-UWKni|3l$No-KqMbanN+r zKpXeLv$Ph|;A#GG5n@83DX)aKKOQQJ^x%^?rZ-eN-yYKKTOQlW5POTlgjC1T7Zqq0 z-;}IF6>VE0?;euXnB%xr*ec5Ecv6P16UM6Jw-8J3t`yYg*%J7(X}?v+XDgJLpg7=k za3l859Lm&cXcLe4SC+@JC&ko^b4bk81!vz!*dlC7JAkNkgFN9`0@3iz{5$>QU!u-U zy@6@!6}hRP)*;I5useufj_9(;J%$7C{FaV=uQ}+HLW?27I!qje>2e}qIv;t^k$ts3 zXu2^!?{KytGY`q13yYN98eWE?pjH*d&e@02{MgE=V&ss^#Ki6!TFZ@#3kWfC*I(4k zs={~3p(B(DyR^~e3s}k4{+^RwNLVIl((~u7`%Zggg`b$ER5^2ragtmU2mX~?A=tDW z_rnI2%W8KXGuF}!|6JaWbcT6X>myoC>S4okVRU|oAi?C}#+{ABEcc3!`Q}p1zLa%% z{$ZA_Bkvb0R&Z91tw>Dv>|!7SofW`X$$9olB4>)$S#bTRcWVihytWxL_D!HAcE3f! zf9`C@&7B|L=>W2#hW(p_b{hM2+yjCy{;nUwoVMs$qV7{)Q{}rk+;ItLM$MzIO*t=4 zjTS)IcNLK+rv}nqJ#*bc#Ft2U*7-}*$nYn%vyG@liGtC(x>B^33f9UA>o@aj3LUts zKhVEitB!cE^^LBA1TKG(z08w;@gh&aHnpV<=o7+oI3)9`#PZHP(L;WW52zP6f5C6j z*Q30ak*W)lGLgxo0d|!RkD}~^ z8U{=b#sbX2G6IkDZZECG@aMg+Ln?fi2IlW+1i^p`_i@&pPas)IMuUg2SiHXO(Com& zP$m&P4lTbt*uUQXDh~*m?zFs2P^e(~y!~bSQtlhPm(D-e>EM!6?w26~eLW<$>)C&D z;K0Zvs7d9At0IbtH+cTrn+V9DNY9#(S9|=?6}f1?=%E&UGk22;s`a!D(T5k>_xMeG z7P`*k--pl{tL{&6ws`emPE0A9;I<_$e*)_WBH;+)<p(RMC}e;cfij)O{Hjo@`| z>;{hJd==Lgk^6Fv03nHrrkd-QA?(n6Fk`FYwj(LrW98Cy4jre_ycj?+-o(%v1gxk= zCbCbYauHr&_qibawZ{TKM!oA~4@l!BcBU&=9sP>y68O(Hs+VIdCmvuXjw_%${fFH; z`J{a3qvs`yF`R$7NP}Jgo6-X^pr)Art?H)I<;c8Z2!UfJ>-cbOCY^8jT^?8z@7Dnv zq#ECP)D~r{+mGKoETvZv+3=rP#mu%K|A7-;95#nnPp+rz4c|vKNCF z1xJx#KCAqfFlT}W6#lyt2D3h3n_Uc9h@E#pcv58hPoaj>e0OJ_uI^eN`d|*4omMcq zV(oD!gVOpJ~Z4vukDWvx$9AjSa_?y5Zkb^hBw?1J9QdO8p` zcLIIxXT~|u`s$mR3L=I*P}anK==JiP*+qZ^qRH-LIDnz`%otyj(x+c9k38>u(2j5J z5SW>2i(Zw#T78}G6*!~aGoJwyzh&ixV8L;JL><^N?78U$m|XvB+5SJj_@f@Y95M9P z9~USP1)UQ!#q4<=^HhA&a%rXgEjc5y2t9P`cNGFzUomee0(uv8B@EC1B-p;tLj%y2cmrom4TO*pt+JA|_v z+rhy7+Ts)#e%z}nS1noM+`RcR!S8Pcqe~fC#>Q_w)(N24-~4yn4ImtZIRo>QmM;-O zK@vc4d~WO{=J50Mw6+(gXiy>!8dF*X@isLFR`kRCV>tMOkC*5^&@9W8Jqli)a^Oi3 zc>I5VB^5p>yum4?6|2em+-J=q5WrhKrA2A8-NoMQCH3-~C20F}a$ooT$-dX+U|>72 zo1-EGqjD(^i4=+l^cU75%_=5t8wmWrd|jIg4y~9tt1AruKYtDt<_3qN4^zWxl-_)5 zZM%+&mC;@$qvB-0EE>5!kJe>-3h#>GKyRz9f4$sIV)T@8eWE>_?mdchlS3 z4XKlJ?{M);fwz_|{9*iG?7ewB)!q9y%5J-niU>utOC&-iBw{zI2u+3vl__(`&|uhi zcNz^12#Jyg+LU?bPN)nGLWT$#5+RCYIPcH?_&&d{`#itr{9ey-DkM zTG#cyhIL)nx)vAT4h357+bGd1Y>ag7=gjn9B^f;Fhhoc z_CoFk_LxHMF@gHY(4Zs3n6i!QIdj>|qx11mh*?c&)1TbO&FRh_1%C;I9+=hY&1Nw@7rdVO%|m zjo}HJs(saxq#yER&sNHfw&kb24)SlZFZ=L-_>S($tbCQqO>8G~dOJMNlhQ>SX0cQ3 z@Vh21ar0@9@u%sYTTapFw0!?wZ5QXQ6l$#$Ewi?>tDeer7tH!x8Tj6ZJ+R`Eq|qQz9LKlt(;h3cyNr@dY?vQn)|Vw3%EqYY_*C2Ld26RXT@5f3 z^NM4ASD((p1h&&Oy&ZX4ZmsiCUTjA&#MCZ7P|Lrl0e{*sw~xWb7^bAi=4a36<};4+ zZ!(FBu@#_&fLZY>Dn~|;Pw*@ZFr>$B!0aqOVvo7x9<{N217{ZKn@G@a0B2_~jo_>q z$ypPyKj*ZRKG>g4vcC%yAvjB5#B>HmUjK<_S%hBC-HByV^IDTxOgy@s?wQ{1yq`j~ z0Z&pJPe`#-`T#S}ra*1tJF7Yp6T=j@>FwnD_nPG1-pEG_F~oMhD~mu9@TU$cX`C!c zNFpV{iQ?wqVLi%$>yT{)t^CNn9gC*c_G$edO1 z`K%-B%PX15FIc*o%C$@KaieqY*(IdUL-yhJHSQ2m8go9y>P>ppv(g=!H-&KJ`N1m){&u)+dji zdu%Pr+w{6y&A%1KW*>JiUdL@UNIq#${Yh>7uYNSHZnai8Rj;Ls%O}P!oha13v!Bg5 z`B}$8WFDTq3=Ej4f8{HL*h$@47vG_%a`9`XwQudKm^uxi(h;s z9tU^wZftB8(%bO)ZhUS>E&o|r7|7)Vw|RY}pS8RR&V9!F2k|^_xbhV>5z<8&YYNSn~6E%CC;vwewWC_>OM+IN~=oQoJu_?A_xRQcewq4RiCGOzwD|Wn-vv zKawI?vEq2{(hy@<*CWS@=Sy$;$f4l9$-CEbNv`9{b`c+Mn zD+O@izk9jKe zYB8VY>JDyw@24qtFPJ1$Q;JZ4_wCrHE8FEOkL-_2ZZc_e#b$&!nsE_UX^9?QO!iSA zUJxy};dd$QkaIcU8PjW0+WH4OMUHE{)@8!byOqb{{pH1eWgexAZ2!WWSR2&4H}Tqw z>^V~XBC2`#UVOw0rsQlRPNGx2`qyV> z{U?pwwJJvcj*7uC>S;%~g?P}0o(G);qRv@MpYfU#wX6l})UwmJw$3Kjq29A6*j_J< z@0z+{kn=06m~Be-;{+eqT+`3!xP^v zroMMxb8gQVtZ;u4A#dKch%wdsYVuLGxafg`@eT!*y8Aza5wgDSxW&IHw#ZU^=*ECr z;gDKoeM$S0jIr?17wKrI?433IR%c5^*yJ|P?~>y>{&Z7HqhqSB_nS-R3)&Xx<4W?) zc)Mqc$`j4~6Xxz}zH3Kt!Q}P9a8sSFz;_6v=+MwkzYWEK1#XE7?G_YjgV5;vG7TZm z_lM7Q9xC&G-JW%+SLA;C8i*_RO}x|6nvt3BG_*7lwffajyzhI1#iI1}G=6@XtfXaz zNgrHFr)^;f7mk)JRgNcK`>Sjb{WaG;MYi}z1V7^?wXI%H0}rGs+m?)%7mk+?K;$## zpD)bRgeM&tv((~yg3(z#c@v*);cW1O^5h_8(d_XWiwT1)Jm9q?KvP|EQf`fSouI;* z!nknLMM7_`b1hi*_&H?CXsH5!IV~ij$lgs>&qt}ToLOdC(D6nx2{H3eS8W2^WK%as%wt^S zn1`o(2Rc;>vsHhM5@&;fkDQuWs_xYvH*LML-PvG#*^}dx2=3Eg-D4c89KTiAYABwb z`qi>TO|4!jtn3VxYdK%{)UJiEmW^DtRk!?H9AhYA;>={3xCWv1y^o6{dASNuf*#f4 z;WP2Y!Xol2UcWd7(F>y<+0suG(bxD?8Mps=m8S02Ebnf9yrM*7D*x5pRHY=~+Bl*-> z?=VK;&!|G~LD4g#oKL(xbGkG}vRnNolBWaS5=<<{`F-P=D%I|A@e<+pQ_I>D;X5mj? zS>IPm-dD`dh$IQnUI!2g|LwI~3ktq7UCs*UcW*m9H8H|VR6T4~=jp^7_BG>bXxggu zpmhT~MU{K8-}oT1;W_(8!9oG^Uk!EgQIY@BaNN%5%*Zl zLx~o=F+6wk)U~DU^`}&z9R~a+UrpK7jzN@}ssd)}<^93F!*K(r3P(=;2NN#OKderav(X#xGCSLee~J0LTth1N zkFy7Hrk=|n%9mOZcRjYA$lHpa>V0!l*ZG+El8Nr^#dwb}^LRLnCjt7)RNo;Lzf`-m z7i@=oi1-^i)bl)d z<4&WkUYQwif1TZ_Y^7D5I4r}xy1pf@)V+6HT5x3ka~&|m{QGKAO0)orA^Rj+;7a}F zJz!OMTV;|>AZp*_C;iFge7zpypljy~xwXNZWubUXAec@(O)$7lktaF3<`ySIeCL-uSft5^>M&0JX8|oS6TP;raNmsH+ue_j{MCM7ZTnRF zHfNEh!sN@&U9kZHDtQQKMs zEFe<>%#4PFh)v`4P`J~_>mA`(_y0wh{YoD8HJ63HOyUrH>5MZh|BdUW zIxzQZ|BJjBm#F)jbIIb#pD|Oar=iWcUitn=jZ!LA90Ndt=ji+N>??-s3*P$G)TNj2 z8CbAyUwsH0Ly$UFFDSZaa#(+A-DyG%XT<+Y_IaM^@5v>6FQP^80N#7g{z?6c72W>9T#bM#}~81MN;BJ1*`98+FMRdTA9D5ARUm3OIPWa8i)%Kd0hljZSzDPTY!lw#n6$HG$%&(9lTV@|2z{sICkCv(gV2v7;cc%KMCJI}J zVy4FVi&`c=w49yAtA4D$zY%V4Fx+0N<%IQBc7zdW#fMYEgcT38k;?to5hqBs^TxeJ z@{?kNgdmT;4KG+gXygI72g*%fJu@9_1T@zrOsTET4$YzmNYD@6ErRvvAGm&BJ^tt8 zA@4uFU6Rj{zUe#mdFkY>FL)x}d-UCX4MFp%u{xDkp{5etA8@e0$uU-DQ_>&NHlkM? zUSspdK68n<%R8sC+d+F>moQF3q18UFfU&84e4~-dRbdKaGdH-drTNKF@mgnr=Q^== z!xQLwd9&_OMF1P4itCyFU+TlJw-mzwz8k;d;N7Nh^3A2Ed$x-SY9N8>a$Grwa86pE zO9U_e`F_Cn50^5{*cZ%A^Wps(2Nt32Tg*9B2Irl4Nu;`(Noa=#@|0M*8stt1#8 z$Wn)dYVrO!YstbY^`QzrzjJ~LDTyV2E`%o3BAxta65{i#G^fV-3OPmd7NdJxtDXKS z6XQ&0Gw%vZ@oM$unJm3X<7A9Z7>sMknzymnpdD|AR`pdg(JNVJ*ci_!e!nvPM1QpS zO?7$=4$i9NR2Zc9+S*7pXVJJDY zboafYJi^JfF3@{)k*-I%=H}p?bS)#?wD^d#0ByAkW2fr6^|vg$IF$~r2+4$>oA<@i zlbRyTQfeRXjGh~VwP@k3GxIV_z4!j^H-)Kh3a!eS*}9|m{l@MO5PsIaWP`Z=cl(tQ zg4{v#6?p??^R`J;QTo1HpWD%1Vy5ksX5-QsEZ9KedWWXI*Mr&eqmldDF^A#Y3cO9o&IrGZFugmGQC-m`& z{dvQF7E0?+Pn}BAN8xd+uFJ`NLm|DSx1&rIAIT~^YpdSAh^1$%->JQ))%o>aluhC) z{FPhSxJ%zTeUGFiKW(+%M^u#Blb#>i9upFN;?up9|8_Ljv~p)TSUAfTuh1C0U@bwU z&|Nehd;|iwop13gxzuE$^$7=|Mcl*z#AIyupPIi?HF;Vf!jbhan3peF$xvqt-57C= zPf7eJd?|kZ^R0(q@b%r_{%~_JuD}VpQ*QOjb7L_01q!Veln;Ey_QPW83u8t1spwFs zZhYtcnl`{we0ok}`1Sx^L-LJV*@qXeTQ~Nf(0_mO3$yseikQWdZ3>fj>ZN>5M&4{+ zB9>G*ImX93UVz6ARZf1@_Ih@PEl6{6G*sME6QL7j{`0m44mwiTy&}hmPt-mLVW82-kxTwYr31qId(P4 zn}4e1ze4CCsbeo!-d|_CbgE_Qm*2Vfy^V#>nFW9Oaq}JBNAp!k$MIH)6IH9G-xXN( z0Jt{BxLhYJm4J zkT`d6)@*Y`a+FiAo5~55zb1kq=WXYD563MLtJ?78QRgfkEvMzVS`V{=*%;R;gO>{+ z6BAE=cEz-;dtSo3;(YjM{`>#%&~UL}&a0EPsgb`r42wT%B8nh=3@&1ls`wZ9+eL5t zp9H~;;Nu-@28~q4nsuDhjaO|Ecrb0HE}E-{X~QkqnXRPG+=H?oK%D6N+DKPpvP7W#{`IZzQ3({iJsS=QZDaMCB(UZmV|dg$2q8l!Vv>Rqa>cyQ%FWaP9v zFbD1c2dK*#K#-#~cxp7*=;oDPlPjgOi1U(n_m(9hZJczcjCz~HA96feIW;51b9ea&CNZV%%2q*8%sn0i&mWdJ_-J zM)Jy?FWWK)C}pj*VhgPVa_Ar)n!-z=j!_aTnKjN!%HX^^!78ujWz=+rSax;Yc3y(y z%F$t6B7YDqv>q>*7K=ilpE#V4as2hy3iZSF%7kaJW&F&09J79PPA-`mEo)Vb`MJy8 zu+myo&vi>&|AHOLOX-IqiU_@Z?DZy2xm=gg(qnNhrw@A0m4%;Amtl;Sjj&Eo-EqOr zBA>?EOV3p1Ru3*@r(6SWz?|KW+ptpaiM4GLIVwAu=i1 z7=wI>h)*MuUkrPWvA}{s-b1N4)K|%Loq#VPL(8=Z9a4uDUCJlAn^h6q%RX4aUVZRW z?K?7re&32&Un z9D8pKXy#|o%sDh#Zq9B=_vO1YiVA*x6&-(M>>0=mqVJ>-_a(CMlKcJ13B!ZA{h8V) zIgEK2UuQp3zGRp?A-#xneLy*+lP+CN$Bu*~X_-Qj4u#ZGAWN*jeGkG93kGTA4}ic>R)7x)EFzUV1dRW4Pu}pGowISUzuw#%0vo!lTlbqr*(nXdh)OQ@Z3tN9>6=6wjkqA;XDr@56 zGv0~(8Y?d0hR?DMOY>M`)3FLSp%63jyM@>Vsa>0p@%bvaR+B;vz&anFhhz6io|fsB zj7A}})!LY!W&Tx>q`aAE=?qN`QpD|V+I}bx^ABo>OCrdG`4Y3;`P~{lZpd1dq3{*M}b7v+DvI;|cORtAaVX`JV5mds>?;tX@Vxv<*4G zWr2tkW6XJa?2&)3NO}7vg2o{TF%eTNl%-{se)A3~+5w#`ai9r_wkk^VfO}F&E&H8*?T%mW}LVldP35m1_>R$imTcjRr?9gln)IAmG5g2{7A8zd zOavgSSA6FNKRd+|v^Ey!#S!8mFp;+U-ZuZDi=fFi5nkZrp?}l#m@)fw0Ln@lm>{P> z;*7uq;|j!SdrHuiiU znBUfT!9L7&$G@?W1#ei*1QgoUZ=0A0|0=~x{{M_}6 zzktaDAm>cdwG;D?45CbA(QM353iehI?8@^$2u}(OlRf{LkHCa8i3tdC(o7-tr{4o6 zs-NBvVyDQ0CW3bA&>i=nJIFCh4y!s6C%3VknYu%3HWbi{Ha!BI0|7Da9u4kgpoX6N+sb6VQ$`g_sT9j;xICon0eG zp&CNOW#4*3ckDtP067MnB#}7Tg6+)I9hVm%y>+!DngHj=xtd))H$*{1?&nFNR!#c; z^dxMe?&S?())s-}_NEvh@lhlb&_GW~#Cv>daKS;Mw;q|()=3|vL zttswRxaYYYC8i-4iu!mrt~xj#xKA-6&<=%Phfsi^6n%{eL~Mr$?c-4dB2Y;(J*ETn z8StiJECncWwBn&qzo9(kXU((A&`78eDP?Jxaaa>yr8N?;CeYU}v@)P26u=RqjYOp3 z1;CtgMGhJq62anNFQJhvmI>fQi}D{4Mw>i^ngacHAiV4oeit<%TCrIWjb0GshvZrq zm;eBR?|c-M>-c8WyWH6&g?NZR0D6Q? zB=ygQ0Ol36+;52&7QvXH;vNY2l{dqf3Yz>gVf}h08FgVa!i14rNdd)Qh=5ABtV9&BU+4 zM*fB0B@qGJS0DbI%3`Z2i>^U>9=ddyh5vBwp4k1n5&+^}_ z6fi%zI0=xSYk?Qgmk#vhs7t}7bK!SMM4+N9Md1oJi>;X~w#EveY)rs6;Dny2DRzW+0jo_QqUbT) zQPO=g$Nc2tv``K{<%&&0Zb_6&UWXGCs{3i1I!66hU^%PyE_x`@;ao zE-;fE0}GR8ve2LI3q%w~-k8r$X@^jIFA0XCi2>(GL|9>dQb)jj|CVsB9?k{)5&{v7 z!|+B)?=2tmlZ(5|7h)!{y9wy41^T{xyRn#^A_2CNh(KlCOa;c$b4k;B4kABmVG>ed z+$eD*1!4tm_fRA-lhFW$Ron{Rx?sU7XWT6n|LxA5s-34}`baHwO znsg=$*%;#>KdI(RA=KpILKB=L5%B`^6KWm^10rTxjbiBDb1diW6y~2^9LnNIHU9{3 zt?gK&N}&P~OcD`zes-n;W2ZpRnN90C8ZGmW)dvwg6F?CkDG&jWC6=MA zd@Dm7Hxp{Pd0>cSO*GJ2^}-Me0ac&h9-fT=yP{#`$T3N%`57z(N`MGb&0hwE$+Lwo zI7cGl8e9uP%@YV~`iIpBfG;|a<-7gB{Hdf+!;q0`{s4&isNB*Ghc~d5M8ppuZKeVv z^jz+=o})t`4rSqJvjTEEEf827%R)SdS(+&~hh_tO`)Kvh?Rluq>mph6+wlMY#_*$t z-s6k!eEGbOX)Bd8VdJO`!tba2&z)+R>|7;El{vXfA(RB8zC}^~^!0(+oQk(pX zWT1^?9yDj>TOU9y- zR~?j~$Ev{^s->^pN1-|+oMO(6&va2U%RJJF+8!Kc%sKyk#J}mA2<`Dn$U>sZ&>SRX zaV%?bW1pk%U$hIVOG;UP8Dis5r^^a%Bfh*7Md8mncOqI1r^ZCyzZF?8SZNcbh!v1= z2DEc5*FCy_5em3#6S5q4`P2WvKF=(>L1CvP!VOH68ln=DgqX?+7Y*S201t;5Ss}nh z4J&9`t=SWRO1pNcl+}APF zjf^BvHOa6lkNX|sEIrKpP=W3Q|E3=Mj*2LJltO+a4;INAR^H@(2dA(ljhSEKiq{74`D$K9 zs367p$uK`X(~V*VcAGJ8o*+K*BtIe;SCSB+$jHG*AHHXf)x+Pd>g+r~q#HiYE!=i` z3(|3jFih_c>`kY-7-N6nkYe?TW!N8RD>OJ4#UZ-@I}#7pn482%0xW*Q3w2a~hhc_t zj@9a{{Uu?4h?ogs?9ym}6#bAC{5OH96$bJsL;oi1-yr}ls*JPP0sCc+==*?JVnl7XTv#RHZg=ZnjCY7XL#IVP$@QoED=n-VgA80y6)f>4Q}iY~-QN#sYMbB?i+ zyVUMXe3UzBs2K%2SKp+LlamnWoJsNzSe)J;xDobou8x0E1@>pNL4P@c_xk?|-e2SR zXP?a$fJ#XzcsbbLeA;US4(#W*AUjmCoR(>GWa#com^s5q%Hz}EB4mqT2RkLD+yOCz zB+Q*;P;r_Y7LG*&;LC%+L98>1+r1{3cnS=MJtJA$W@-<(`@OdLCHHXz$vo2CI(6|; zJ^7I^=q#LQQ$_6o+dS%8dTa&Y*hWR!veu(APV%n;@=Wdz!L=Ixyu*-Zcw&{ix-JlK zfIR;@Ji)#&%uTWsqHFq3(Z&99vA^XIX9Pi}r3Yfa%kkF!Y>@n=1zVUtxY#_9OR$4Z z_dGEd1&>OYJN;q=Do)&3G=Nwp%@9jMc6&+L%?Edz8%$LR0cVl|E=S8HWS78C(+n*z z;RtWa)_rIdu^!Nw)K@rrOz#ieF!@tu72pXtMSNDrofIMLZw5R;E_mzHEUf_?YOHP{ zv{jB}bWj%Ff_8vAh5c>yr(ePtDLvPq9WGQ}lR;#~^e0czH}2zmpuZ*g6FF4xn@gByp$tnd8NaNAtix1~`jg0n{t$3MMex|R_H zC9_HQ;Yb3KOM^XS^c?#~{szck27;D0#k6}8od{Eyzmdc%>Zv)2f95Iz)Y=+P* zJ)1TMMS;(gUe0#Trq0GfITpO(=h_b)A|spCQMC;se{f+RjxE=*f_>zgxJM6Ln_vBb zIyo7vUAtjD){gULrOsQlu)x}LXRICfPaXoA_O9(nG#LU5hMQg?_zpi#Th5u4&ZOmb z-afhn*Sw%{R;C9cAE6G-uD+lzj(}piLDNa@1;mXq|79i=#%zx9$9oUIk&B?00n24P zN5`l#_&Y(N4_EOOEqAu`9Z9|i2(23!9xIpJeI8dE?}mhJid_Z~*z-g#l)e=E57}(; z2g(+*BFhD^#$_t!o*4HTlb)IIL=81>~CYsN;{9A)veaZO+ z!8WrQY~z>S3jXiiQBK=^1^j>T9>p7c*guQ^))3Z|KTYKsaQ6&ZNCN+v1pnRV!;TAU zsbPolSOK{vb4(mT0vZ{?_^Zmu*A77k3|&oILfnaYo-||a2)o)NY`_BaBlr$XgV#eu z-WhkdXR9=&%V(kL^k$~3c03mOgs%}@c7}~K*Z+} zN3P9V-Uk9)2cT!H9V9b>mSX|h0WPD1;P7ddmSd7yP8?1k01Z%RLsRT+G-hczXF|)_ z!)Y`%EnyN25%J++N$u_;URjnPA3q4Al;?j>9{|WUU`ac?*cog)NI47sZ=P+ph$@4n z3JNyvCHS97@_!d@va;kK2?`k&lGfsfz<)1!%+~ev=n6t)Uw225Ytq5Q(|`t#_rEPg zwguS(2U5%7>hH|8gJkdwNkFGV3e#tw)F8CnCqtH&1EA{(Km!!<&=e1y;3ak~G{Hki|^SL{YEGl3=T@Nj0ZO&)67%7~@51+mnN z7o8pw{5K`}FGb5G)HY!yoM-5NHDS!=emD)$<-JIX>XK`c#Ka_^A3_#n?Aj52L(WKv z)N=6v^o+H$0BuGB+62bmgkvMJXzu`MmX-scdkH|ZVY%kmCwW}Cfj0K*rfILtEg1@Qvi51_%1w(g3ecL2d;G$Iy5hu-Xpz>wSY&;y)8 zSznc4sIAR(10x8EB}k|$%~TL-D#RkrWwYZJK{;q@vlB{V_fnPj{)8qgPj(HIeb;U3 zSu=4pzIV(-jKL2rbip89iqZjJ^P7X?S*kX?ns|eEC%N7sdfoFN)qr0bk(bc@x(-Ws z!v-b9RT=J#T?aj5vWF_#vhZ6U z{ML!T?ZR){_}f4DHy=qit{;tM=WG0U5*)wq)j~mT^-s^wn3(o;Y5Ew-TDJ7F=Vq+^ zTBo0z7wX#oJ$7TmqLI3w*=0-oPRm+~u8S@@ynV?{=f{jIdJ~@_W@WxC9P2J#(%7&} zp{Zxwu(bI|wV3Im5%E(KChLEdO;x{q-#0hsz^N@p>%OUIPMp1H*!XpC|HeR{OP zZr-e4;;m^@V>{dSxQIE2_hs5yH+^v~C2$T})nGe1h5C~3{gj4UK!1#{!JcsMh%XvkuV2Q!0A)WHWgl|Pmqw+8pv4iM<-2dK z0nwGwIK-{1#sAlr?ZF`~HOXmtV8Le@^h%FT*J?RltF>(RRezi8s|j{;)E>k_zQ1eG z7M4W_?@7U#Q@E1+;K}Ko>e9OdiC1WojzA`i|h>U$2DTiS;j=Juss)ik(aiG2Tq_#V%R5j=h7UJO~VK=O6 zzW&!U1NF`ad=+ zjm@)P^&>~CrDUw(lo^ctmxRn3e(6m`n?fY_zY*bjhd5XdYiJwYGj#9HFSOmL*Ch6( z9lHR*6_8GX3>$_9#^aRoH+f3gfWU(tK@!)XI0PPS%^G)rlLiOlTsAlmYYf~PyPUn! zP12y+gzKFI4k!=PL*)$-c0S_6Ig<3oYq#Z-I;AoB+Pda1{D>GCxKu;7fyrF<;yNx={E za9}g1$0?nv&3BB?irXOV4~hrkGUO1j7{CAR24Y{C#J>2Yx8dLg(vcB2Pisp8GsjqD zxUL5nh=3$^H`}F&h14lu1y=5}IEHd0Zjy+c!LF5s$!OR4GUz(zX5~f&28pyMS8>0XN7aWRt4R z|9X|tUM1#Ph;j-c%Ab!Q`XgWKb*x{+hf_)D6qa_xr!QX~*z~ArwCBD~`$&7vpKx?8 zwFj|8atN_xLe7YNITQPm+)GRTNx- z-(J-JEnXCn5SFK~I=P^4zGk>jJyA8fm-|`o*vk_ZI!aA%*LV4TE(}kqbL;p!xWGbI zXW~_`PkyNDT$6&LGr^}%%U)`-O?L8mRdexl&Zd!H-+$c~bv$&Jnfc!NV^|`!bH%Y3 zwLUSusn)YYjkh=AZ8jPTDz*I`&)=okMt}Qxx4lOFeg4;)s#h0Ba>8Q-b^JbT96jIS zWPKqD9mHhYOE+mwmH0VW_r?Vu?fQ}xT9TpO{q;j|@s?Hkj9?=ijpowD#C9J;EP+Zv9YU zlCw3#$%nU1>4SFmwRt9{hIONnNqy`0Cx2cyzwJ~H9Ng;-=GCs1N|TQk)|pk8yehRz z_)zbIrFjW;ZGQw7WLlS^17)9?e@ZrS3;gQlFAsVz{lz zg-pULv^Uk<{w}Qg$(?j-kHD6Fb)Jh8>W)V~a+bZ{-eX_h1AaXz{TILLTqEsCxAI}1 z*N|&vdcJ1O5BIH)2JfVr#Pv_TYK|^4Gp+p9Zc%yVa_ITZ3pTgL?p?g%=3nbOe?C|D zvbrS_@_ZGVp@e_F@l?}leZcX%zUTJswAsFV+s-R@yB|)aCRG<$pv}rg&AP`)>CKHg zukw?9<~rmT)(zE8@SE^9tiio?Cm>(a1A^A9$SLlV#2g` z5Uw&S>#QCKcl)f06J;X^w^1RsH5rGI+>3XgaESPb#>o+IXjJ3K$s0(`KXA|W?pt{& znZC_;yu(60*X*el=8qp#@#>J)R!i2&^Uh5h%1LuKF-)#VTikCq?3A2fmLDW7ci(cG zPI(W%-+lSM4g0%}s~z}YpjTEO8deh@4Z2Ad7+xNd|J>Q{s#a6m-5BJqu5H&}S*9BP zV|;(Zm-EK$_dqw7{hp}EHloDm|NSBiET%wNJeak?d)69ikW^XH7+C3APZ!*QYysy7)jtX{{ z@B8^Mt#1C4Qe9gdW*_vT&OMo`edea!FY5|+LztY)qPeTMyg`AMka>go_b{>;iD#Ns&el}6_)i8f~!_FC>@`#vZU)^ z!_YpvWC(6lm3?5ro_gFwHs`+n;}4zWiZVy0^QkAM<>|l`h^0g2FnBGptTk{4XObX%`0DboQ>(LTpK7spw3R1 zza;fh+NBGT1tHRbh1s&(_cWX^Yb$AtkFEoxQcham>+sf(b2(+u*WPI@Xnatwx?!%jHX^-S5_w z3z~RWOTBhSh3XJRWKsq=P23@3ilgG1hBmK9Ao;%7avsgbC~J}JGC9)I>2CK*KR!R`ypr>V zM5Qq;Yup(=DY?5rbXC4g(BCM~CxR2kkqt`_jZk|i=7~F%ne7y-)82svi-a>@KeP*z zO&hqU8{37WXBWr1aRUgQ(Uo->I5b+Ol@J>C)O~umuy2M+`MRL}KwZ4qjs>q$pfc>o zHs4t{*_pI>8>H!iuuC{Z%TuqsCp03geIpiVke*mKKfdGmqodeCTmQRwvvWN+omV90 zzWlsC_iYNa%7)skXLT1{n^ zGu4ZbCVquxd5Biu&9E9-MY*z5SYV>PQL=<~`Df!d*uk@whdR0QYE(by8G40QyL!f3 zL#rHruw0_Qk<_8%_g1B4E?cTS#8#5mW}@a8y`xyb9p?R zu>KF!*c0(KOFMf%Y^Lw7=ldtvV-QcsV5`!Y&B3P4_wOA@di^v?nQ;r@&s)N;2aFIF zGTZj{JteRp@QqNRB}g7kF*6^nZ(dcselJiL>bZ1}dO4voKJ#37I~tYiwgA#3X`W{W z(P~hYOEj`E2^v{9mSI$rWWN16VGFT?3x)f_J;f5-bD|%8mAT*Et6kpnw(7HSJHLf& z>51^;LnB$yu!T-#s^al!Bc1Ixbx!%p^m%N3bIf#ZK!@5s^5m{x1suN zsQzDYqC&eNRtqZB^=YTA4oH7o6#ey7olC1guP`1;W5!w^M(iB0pu zF2KXtHQP=E8IXa$EcT{XhN7!R3c6}Ua!)5S&3$pl4G_JcX*d@j91l^B`i{8q+pWz?<%J#K3IVd^QNnSG_H?@ znx0Zz@fxj^YU*At@(PEuoBPho5nbT0f!uSIz? z>=)j$$tE{Y3%~Nqr;^3H<6PFnJNw)*m(9@o>QA*NuDQX<=EvnBPxeCh8XS$NTQxvKQGHij58&I zy)930UuZ{>z~p8}WnBHW#7W1MeK9WgWut()$x3+#al@;PBjK=jWQRRH4b6Q_5Mf|m z6LzNu7L;7i`F`Of^ZkbGS^N*oNH2B=ZW}d#Ck8CCyhcYR`-Shn;+d*?b3OM6;Bd6_eX-HZ)Kcu&2D&4QU5`oN0Dm;q;W?$)XXoH zz(75L0TqTDD3G15;!Xg?lmLns@xq#pPZ_S}E@jsr<#|$%?j=EBc^StsTxISAHQle( z(_G=MUl4u}&}tW!$o@ti@JHvZbUpJ5cBL_)eLb}+deV=4`800_pumb}EJZxjUKl)q zbB`<3#YiU}K9q}2?DMe_uEc0uKt!djJzrfCi#*N*c_MJIuFNaCw;g-bA5}H1xykhj zOLh02p7`tS z0Ot26_NBXG9f1MQiOO5RfRzz}0qRRoprf|Af&hv$0hA9!id_D?7+Lr{%}QOHGtUszkTQ=I3AleC>5r|sfWHU!wBf`WP89p@dT1K+pzb2` zni>sIAOi-_F%QhZ6NGs{mboZH4q)^FW?D$CLK@IM;0(yK>0mBiU7HVK1$k_6Jz8am z5+%-9%%R@eWo+QSETj;3>RAX-!F>(WZI#>OTqe6vTk%_2{8ko(x%jOtek+UL%HsbM zo&_!BW>NH{YolK2@g5bw4=-FTouh|7q^hZAoPj%aXb;2bt6TN4S{Hq@5Xcy3YG(sA<*;sNx4Bg(n*dQ|O|(Kz;@ajh+!^j`-zTCUiggtDp3A2xci$@rt> zP(^1-2Ru$U2bWIj7_Rz^u|B$o0-@BeHoCE|@X@Op#UDwaf~RDE7CP&!`pE5fc!MbX z1@t)X;cG#E8)Mv3l_MO0okTra!s0)+z_%lN)>@^*6E^Hv7m-dem)$TIn~cL>S=L;R zpx5KKd&iawpZhx$%!zXbwF+XzQ zo-ypOI{C_cqKhB6!1c@w5q;c+I=M&c3_2F#io8ic>vqaUq+lAWpDsds&uw5xWnmL> zQgXFZ95#7E902fGl$DXV$2uMq2R>(BO>j!0enUa)iFa>Sab3sFOKrj(>@EcG{N`5r z)9pYC{F#dwX8*#v{cj>T@K=jW=9&4l-Dq?1j}x8<%Gw$iGQ|Yo>#?rA?~%%t2rPIJ zR$3&qV8Fz6)A}f6EYSXS(VPNzP*S~8eV-U{)|byFD1shjGRchHDKD|xf*Cj?+v9*4Olo6Jgt2o zcAjnteu>7VTY?CztJ%sgt-Np~tbe=k{huI$JeRE^i|A(4M8_eV<2BLO?mmdV&zBPv z+5`%L=%5h#5O`?y7lA@JlnMQndf*Ly{gQm)3xV$Y5K7)#*bY3_~F98C}cSe)mnL^PXQ{Oo|*zNCJcCxzj|4B$8)}ZMdtw%pT3) zV;yi_@ZeaC3%Vz8(#$(ji*pfLJQ8Dt!C1sEVAT6|d1{x`qHh<4>I4 z{w`7Z?udzP_Gt^C8vv0EUnD%z^Uw}bQ5s+Q+M$GX3UqHpgln1*5dy(8zb6ghi8Gg!&@B!Rc#az9CQ9jraRz*2zr7<~s!wpOC~ z=yBW>&q?xc7oe?1AE`*AiZRw$CNVa$2a8|@6lMz}UNg#lXpDLiZKqC}tS^?@jibFx z*66)kftc7)+in>e)FRi29Jl#KB?K<&veFivWEz{4REE=W&z^+;RUl|*jj|DiXdI&6 zb5w}9{b?3&&~%!6*a9}UFu;b6yPnww__NG;V5nW*dZDE@xayKQ;zZ`Rjm6oYd|dU}s{^ zYCp;Z&Knbn5%UG68}~w)P~Q^uzvj9<;#fn@w!UNs*yvmqfkE`%X`-WnN)8!@g1Nf% zpdKxdlfd0N!DHx8&bnyU0BMDf?*tiA}t6^E!RxE-?lYLhVT_4};^Vq(}n>Of* zBHZ#W740#ibXV%48OmkICsK%s_wT(?nGy+C-jXcH3I+iFo+SLy8A#?~L#H+k%`HN= zi&g#!dyqod0|mw;)Zt{^>p6tc$RvzLEjpuS&LjE-#2O$dF&*54L5rmIVd1slsl&T@ zTVvONUx9BL)(~M2qI-gWHk_ZV}EpfpbkyH@oq{P6p6&h=vGn=oM)4BPw&;UbK z5LyS%koTCDTxRuSS`1bOgWtghiZ&E$@H^#;%Ma=pM0?CcwOtC{WRvTz?7RQ!1@I5g z5`U0H{sD}}7)wuQ08Y7+Hib7p+ma@tTk<&oRqdG(0{}k{jp=WV9x3F zfI??KJ0j?}9o9v{`y4754nzO1j>hAI-MgWR&iB7&dApg@-Y))t^nfeSil$R4l^q^1 z`0}55#YA-OQ!Qe)GDLi~b%p7*=I*UxzdFl9?YN(2Z6= zJq{h0qu|G<{KJpGj-|a!Dy&x^T1Bo=K}U3)6;_NBUOLtT75elz9w3XAfHcp~Bi=!T zg)q5M6#!{Qzq1V_7;t~#KZ&oxjs`*Bkug^xI6mZe^DuU9bRJqRZU$(UDvJsdZ4(Cs ziiWunT=wPhG@E>&f`&63jUxkS+{;mU|7xuJZKte9S4em8Hv0wf z%1jvmLXH8%NdPgvJBP6?r_Zs%(b?%pQp_gu3@r0ICjF7SkR`V0Ipz+M6#)7L1{K~! z*F@GS3#6VQllnLkru_3rh!#Mkq-VgCL2&h<4%}v{gG(^0W)+oaetevmlU&bXc#b59 zaW?dz_MeaX4F(dN|Gk6)G>;_!LM9o`!ZRTg4|p7+W@*6|m;m(R zH#uLab(XNZ^%DsJPck8} z9drJB=IB=mw6T1r4cc8N31xjT>lbl89w&NM@s+ybP1n*zmS#yODjfn&;nlxZ&&*u2 z^eW<+*JI;@W#@Dh@{ji_w~G(_rKwlyh7Rf6O_M(|d2@SHbh7FzorS8@`txKmi5DTC zM7>CQLOhXz*UE@@#}DH{$^^GBZI=}0v7R2|UU75v4=?HLn!Se?!?s3Hi8t*ZH#+AU zeDVEa+A>?%+`BcQ4=s23d59)p!vE_%F5oh|b9mG=%y?DUx@HJiKh^)87{ZQfoFTpRP z(GDEv)WCQepL;j)25DBMlk>ig_rw!`1%rVue_7zsNr~+enOZ5v&~cY2Duq8Qi<5G{ zn&j)LPmK>1^ZJbz6OU;Ii7}c)blM4rfbaic@4cg%?ACozMWjfz(5nSes-S=f5^SI# zqSB#3>_l96FMQZ`@BT=T6))Bd!KX98TX(4 z&o{m?$LkDpKE2H6cXPDxa+}U6hjFL}KBmeUY&PrDZ;3M8B8A=|coF+CUB174sc-<0 z+H%1AkDkwUqtf42As5hxs;#by$iQ|NzU%``v9=B1y?$>1CGMDN#^Pi;`}fP~Pm>CU zVmELa->54xaI~{W49?7d^ zCc4uA({lr6TJAuARypke{W<*j3Si2YvS~O0?oA|MA>F+6^L@CFO>n8HEuhAWhLHhq zR;YNw)}A2_fJL7@s{=T86d0t(AER({Z&?Y6an26~7dXyn>t9!&u_jxl}bLLQb0 zfy`|M_&Ytu-M)uD8FtS+n;OwXdsaFY?lTc_ys_jT$$|6DhW3?>rGSkd zh`^LqE*m;*h0i|2XI49Ik`YR1y&eY!6Y}$LvhJP-5Yt=%#Je_&<#n9!));m@1=A2i zaY!!9kC)nHE`cK;==k_m(Qe+bP+Sn8*wu{qz^|p_0wy!jtJcAU-d*+pv8fsycX=0T zLUz(R7I5!8KBOa4mX8iTH%~>sdqA*r{C-Y20Aa-*z{_(Aj=RZY4@hPlX-Ng6U-9dY zjq7AqU`TQYRi^KJS{jS4evYNa{J9RmjL$4v=&%>)P0p%hDCnb2QUuU*6DhIZ2WLMz z9PeKSP@?N^FDU{lq$z$_6#LA51$0Fa6@pox%96bNS<`n%uUw2+X^dO9M^C7|ci+sJ z6RsIQZ^Pg)W@>)&x1c}-%M`yn%VBi;#LQ!i*Q-H)a^{qc}xycD$X}Ta?SOIZvvcjy)K{r zy#ibaGMz4}F2nB;3Uaw8%sgg4n<VME;zZo|7U+bmTP}<`j59IKW@JS*W`gd!5-X+gh*(xy@aiE4W-mY_A+m1tGdO zJ4msAHv$Kl5_kYxnEuopEU%Fuu1aC1X78yL%Ov-nJ`=AA9UrHrnTNr>WEipjG7enV z%Rl=gG%>igeX@0x_VY2$5ylbDijW@Rp?d#VPBwZ4{beYi&ZCsUkn77s&j4>Ecr$!#WG9)rH~9%To>|By`Ky7URv! zd%q!l#+E)4F#r$auNlC*7>YwMJj=lr46zcw>(H z;_QotS8ZyE#4p;qkg$F5F%1A6N29&_CYK{_do=OFAJJbbD1u80g5>zf^M5O7||Dw!jbbLdQ?mc@D^`*p)BvG{3XL4 zP;l{G?(Y|5Fb<)k0tMIljJ!)l*1qPh>@Rv|O?t7(Pl7sjKK(`jX^n(lUqlCyW&sU+ zEUW2)LI8tg@gW378Xs3zErg|f{gfIR3=uGaCE&=1ev&rR)S-qixW3gsXukU*MtSAi zj6!D*b5A&W#oQ&P96^!KBI1?imxGew(q)_@S{K=%#l&EUDTeUhbB1gwSFrC4UOQNqxDrgPf!852tZVV5^maa~yN7rB|w)eeq z@uOirqPuDOG9hS^xlKIbPS+t38Nw^L_ZY-oI!z#iGMu?0jhhq<0EHZ828y z4?~?1y_HseQXZmFb*c`V9F1TB6At^jX!YJz(ad6LZ&JAc2KQF6J6x`9zLa?l?4lc0 z5WnZm{05)N4QOjU0ZjJ5GeZfV9d5ijUp~kc_b{n%0o3(fzVP&Iw_nLZdzz=c=o*d< zb?5M!LeEU}_;24Mzf#<8dpB6spnBPkbWb1mk+)I0sOrUGM4)stS!m5847bpOoAoX7 z;~2PZQ=02@n)o18CHj zaTsmaNSC$e;=oowq^ajg8Ist*%1%hx+WFm*KQii$i4k~vjmz%?IM7KtEQ!0nDfrzg z6=YPBnaR!cvCRjcv3M?NnpSDb@AhQDkZznikK|fdoUg(d%q2;`+zbPUpL|4Ml}8OV zC2*wP4N&w~V5im7>VA{xV;6v=->%rMn$B4Xj;NLo_afje7k@Ca6Td`)k^5oPy+Ow+ z2`Rk+?aRbcG^e+^@=DT|O9)vg%0ra;ni8K!|KqE(^>QpyQN6S&og)|AQgJVCrDf!$ zzC@c&-Q~CkILq(bFMwO%wBZ)e5X9t2x9=j>*y#4Btvas8i-j#2j$`izGD*~no?Q%L%=@1`w1Ghi6Sa5G0#^a zRV$7S4ES#np%~toF+u#{gc#|axTyfidkCs|+8`&^3xQ9MUHFh3(4>hvt~j5f8RIbt zbK{e;0qVdU>|qE2d|4Lj=5+e$gb9FyzE$}$`&VdifVGtGCyiYGiTn`_djp@NCyMhA z_@*Lvpu1|m0;;0V${7S=4c*~VLn;qZ zb?)yk*$jtj;HCw}X*o2jG{eF2Z%IS5La5eP20-u9v#(@+xdAG4#gi_X3| z)4esMk<Q0k&s{y>C@PW<37tsg73Jh= z#X%|cAk(!XykIF!UGG={Y4=t~>-%*q01GX`DmmBAf%o6)c8JoT^8N0 zeNzwgiuWiEpJ3UxK8sr4$-mrMjd9nsp2~1SgY%Tea_zpDV_JMfxi7RfMuj<|u z-K`zE02bC1^y6BVRe!cM+x@MO+!wn&*QgrQ(Y=t@<1escb`;!_G(thIm(ldPKZ~9` z0!SYbx6*HQ9RVYdM^Lf0(#W$3grHcuBi3tFm3G*9)ENdMNcKI5tmZ`;?xh!pRS*PNB;(yW;sFTl~SU*`)Nd?bni!#aciiE&*35>|0gAyQfH1Yzgx=}L|uRmI-zc%qKK10(-3#7+W} z==Ej2hoG9O!FL5`1*JT2PSf);2@D~}LYEuiarqPDz4~HsiD&#E<{R74qYw&SA&uWa z74_DDdYLWj1F}w4qt>K40i`~JKf+KnxDzULyZzi_n$=Co->pthZ&A=ZHu7flMnUAo z$BY5WRTRZ0t)zcGRCy0olcTl!^S>m67n}gZfDH$MWcdF0e1Lx6h7$m8vIJsSnWOCr zZ;`%7;dU_}8;-YOVzF0(nx40`a zQKh_y`)_scw`nuqN<+Awy%K#WNbi|(`5sUy-}01K1D>(3c^qW$g&ezHw2UU&Cetjiav>U%pc9DVIcLd0|< z+}j^*bJ>UP#tQq)r=a+{jH@RQ6?!s$a!k!aZ>LT+gc#2l&evrOj8(-Y|KT1Wd7Vng z>rlYMU3XEn(N)+}1#_@4+40yHJG7elGnFB^r?s`kQoV#_`!X6`tgz~ADe5NMbz7%XG4u< zJxFva(@?pbZ5`i>gg>mogh3wE0U-$NiVYyIa1=x;6$FZ~hxM?C?+_!{}K9f#ytf);QCbZj8+5?a) zu*ipahog6_hDok7%afp_ zh?gMcF;u&|V_&1dR` zsM%36JkPfF5{;$}pZC&kAJ1(u!DFKt12?oFqREgDB~!`SB1>`Y;D5wV*O(`Yh&a4; z6-eJSJHb4hSXbTh05Xsby53nyoudhAVbJ<@)bEoVV}-qNxQ1gNgxaq{2EcgjW;K-r zvbZrp!iJ#?Xldxfs%^xM9y+$z=~TX5m3Mn@PzRIJE|IWvcDa|hs-Z9K)kk(io9!kG z*--(;Ze21q9~xEgR#G{@=mG&P#e*_EdXNQs@a2JMO@_nEXArEvQ|4)jti_y3Ggkmi zMEWn$hO-!f@Pk;D&xQb^odC3ZWIeMVKbJvFiMeGVY z_I`&QB#H68PG9AdJ$HI^?JU#L8=`;qQ<XbMFw``en}t^~GMpV|#zZHk z(@2|MG?hOA+be}=O34!4kj2%nY;2v-?sv{?YE5@#Ru@~SlfN=k7p3IxWMo-6z3e0U zR)m`YirHf8tGB+}G|#t-waJ`oF^9&Vy@%}W>WkLQ#30dxq(+)`%({8GjdElr^2f-_ zZEse`mS4Q-lcGw5iwkpo5Af)L)V-3N)3T#fM17VKdbIArG3b#3~fxk4`zP{^OJ0U!NRG`&cZ0gb4p=H&!L( z3OM=^P^~#j;OPlv?aU1-<%l)wpiZabkH<@5&1Jq!Rxo)ZE1$rWg|d{zZD)qPM! z+94it6}E#qn@0b%;ASd%=&7K7c(F*AFPbjk1CAay`Z6(G(LTm$%d%8gJU&Vk?Hf;-g; zj#YsIH~>UatCR+0X@GQIKp^hvp@&8@>p&oMDr!TgVr0!WpdfTv><|s^flftu|3P60 zpdjW?3f@sp0ETER;{>cAbGZFK-j@MGoQCZlSNp?u&=2s7zW}ufVTfPtqDYmf78(My zC{zVhS;|qh;S8yfQUL8;1>|y&2E_(v82{@G)R*DOQHov-3Gu5R8E*JDa3h_+ZsY*` zJaSaHV8G|mt8RtQpaT<;CV(VYA$(Ti7g7ewXp-pJD<6TLf;NYw9r}b3WXLdV#1`1y3fK!8xuhM%V}1RUm13WA0JhB(b&egzo<^2pHqmLh0~1o$lqL+pg@ zC>jD9rqn3>bsDxq7(xY^g!1_sl!gy70P|5e^C*yua#YwYMYk#+$OR3aRiCe1K$s-^ zuQRZog-2FzO?LNz`h(&+)5Cn|!X1U5r`$*)RFILQ!UaQ*w-KK?;tK+ikZjavC?pDR zfX`DyKc_H7qybLvrP}qaAkxq)aG$N>_69>>DGWgkqW32Sl?5*WL)^T|2ny;w+`d>S zl=Ofh?0zu>AjtjF5TNdU@fSz|!Vn+Nf}6AvFbP@!^ZlYKOqQh_H5JZ)ujK>`_dX!k zQLZ|KN#Lk|oB>c-M@23eK$sYw?HP z7hd@#(xAsEI{q5`{QJ9{;5KQ2NK+Uh9z^d?3L-kb=$DSy1qJnX8oDfiAu=co0Xp6o z(ea4MaQw?;aKfWHw|x%-Kv6?)If}xWeLybCQ9<9lHQ?$4nS#l>+`{G%DZv>qf1W{E z5T2DqdIp4mR{;GpGGe_5jtv}@aw8$oB(h-`;erbyDXR4n5b5ay3n*=ChvJ637vFrg z_IL_^G(EkIQ&g9Kr`kKgz>~k=T=$ED&V+sgwRfD9HGuToSbGi^@!yb)e=>Ie=3cLC zOf5pP8gC@vBK}Lh{g*T!_J_{uQ;{J6)Ciy))<}AS71la`twCSL-MG{N^K|Gnfuwe+n)5iA3kU z)*Jdxc!lEXP|!NF!NH*iyL0cs)dS!bLXWWJHDnE3BFulsCHjZL`!@#Fk1}893MkD{ z!~*`84$HqZ_DV2NS4sObCv+Hr*F!1Vz*+`n4Xh$1>i;LJ_{&d#R|%oGrqFbNxVi>E zzmsC@H&84;tASmBL+=m?d<_o326E@d8oU+w?>Ghj5cz*+Qgo3vqK%f66vPHloS1)O zvj3&IBSRM-`J#|H6xRx%J!qrn0@kp<*5Cp6-*(}*rEEg{pN$t7_u+w!CPk<>u$=!# z`2R=v|3~=$Zwvp2@54o(lUnGo-2-7g`?2|FWbQaY-^Wy6(*>p|(}fnDEe26rI|!D^ zx4Xpi=!W8Eqi-*~D1ClQuLajVU!Gy-wrtfE=S}=D6@@ZYbSOGEMl2q)yILM98=JrS zQWQf+P+02N+45q+%rLg8s>cC(rQFMiYTXW?C^?{4QNlA%NI9MX8vOza&7mVz0&J)8 zoDGD>dSFmQJ*uHG_YySbzI!zRO&x%U`u}<_1isF%rz>D%5*+iOc|#Poi!APg?O?`+ zf%aEdA%-8#!_WZaO1Id)_^HR7c8l5*;s+zxUmeZiG>edM=yY(0fq7= zP1rM|j}qO+`q6bUb{vygW0O6W*mfzSMH)FeYnB~r%lG=qhm^eM&M*56rZLmYwNF~& z3yG)|m!9;vRp(yUh-UO>!!WvN_qvv&O(Qc93xS2imk`3RNm{> zkp`dot381oo%>c+?a>UVJ7nA4Bc_%Ku{_;t52C~}ifUL`+{bs8b*+tb zUtMiI?d`I6+6}i9dV8D}b;!kSJ)59a76WYum$%G#v3O~a@b^>*x5iK_w;C8*@&{$~ z_~d-+hzr=R6a#)9yKv90G`)<&V|P6ivAB`E9{C_%ZVYLeP`{~55}-feo_#hGlo=xeS~QLiFl|V#OEsD{`$R#7dmesNccV=brf<8V1c(EURP2~V--+qV&XN05yX97OMP(J$rzGk5!4TSM@;LTE1?xE76Zcb@j ztdcA8aIa~%Y~$(fPrAA~-@s9G#;w9ulSRz8_O%U9mFh>Ev*v-*X$n+NAM zgH0PK8h2fsdL(>Yrow(7pFBD*KdcT8Kc&g8hT=AOZTjdlriWj zDviZ>9G5Gpk|{TjTek0E#}?S;JJ!Fv-7Q~AAGcIWWEff-jmqKw4i|_FGP$nL2rn)H z?!bE6qaX-rAgv*!w_^+V`7oaTB~|}bD){b^@wsP}@ep}|4d4IM9j0wuhfB?VD+N#9 zs^C(k4zWWf0D^6`ydyM_&7cPNc60S9pa(phc_Q}k1kmFIWEeh9vjVLcmO5c@s?K~vjAONIvWjz4jzA}|_=Vq!V zh`WyL4u?wxSZ=k9Jyj6&!!s)xjha4cU4}1R!$tG$_QchwTtcr`--I1|Gr`RNc5{O- zx6R{DG>)x%b=1F1G|YVw zy4^fl1Y$orG0Pw6yxB2PwzxjlKlM4Wn-5c1*yiD0qCY7)%EgnGiLHFr^IFr_pw%vj zFU@}O_+q+gKm%sgl|Wb>9M@;I<%7q{;OA{TR{j_QT&Qv&4dgQfPMx%i1_HkjumF@b zh@I6a-!Z}(&Av1U5njO>W*-gY$acZ)Y^^B=mV@NP6Vdydf#o0ud%5|u2&_SV>4`|t zCm;&MU>j?J)Pr_&K0Q1HZj6_|c$to#G22?-b4`qG@2cY!s5iNwZxVV4CT{MnGb(b- zrYo(g%}V@b-QnKOxwTPBYmKbYnu{KDePu+nmz{e@ovC8+$jQYUx8}Nt0g_A|1lQ!L z7Ig`FqHq0)o-#t50JGvui8+tYkTH7cqnp99#;0)K=GoLUME3!8C@IJ0tn?J1O*k4z z_0>G-wqLp@l)QqLmBzR4?q2T`kf5{oY@8k94`PDXf6&@Lw(Tf{)ek{So%vw{sM%2v zS5cBymUSrzWwZFNZCEHkU9YALAaZP%KZD41?3^pV04ZVG}b|28j6Yow?Rms z-5*rkU0{YgeY-)$K|TO^8V$KEz*edHcm?Y3*&vPZN*4xW?F4D0=4AoCd;@%$Ek*+w zKx()`M8#Fnfi!wWD|E|`FW(amv^HJz)X5<(URZNII!QqBN|cUg>jb&C*Re2mbXf&W z>p^CrI8S*{fBX_Nbvqs2Fo$VbGfwvK&bVJcmViwj!h^{h#?<hLh2G;HD2M&JQ&+| zp`l-sy~pldO8b&)W2`9G=G=69y;@OeJiDVSZ9n1Ry89(kXHXdC<)b0)dy(|}1R>er zw}%r-4Di?m^Y>$ye>{Mpf@H(5@4y`|$WMo>+5>Cwz#rbtABJfEMM^qknD)T+QP!Lw zKBs&~1#5r)+xQ*NiyGY!3))iLoST}|sL*MPy*&V?6}8rLcdCFzfX~;rYEdssY3{Qo zgIjwaf6tHH(46;eF8)li@AyqYQjtQjfWkzFaG-hjB3jo5I~;Q^6u0QrZ(hdlMbw%g zr>0?2tBxQ2YG&jh@qTG>y?EhK)-0x|K6kC)#;Rjh(P#soTepxRNc)x574i^Dcx5RZ zZ8S~V1V8b!p78O`&pFG8$C4-P1I$q*zfU4Pv5RQ4d}292m&8(2T$!N>`R-f$)lh*U`_?ZoZ5q%iF9k(iA%G_b&He1jf?quX!v{933TTD@2N%2^mn$!(ftFLMc_V3}Qp^wIf`u`j_fVZ5>4i*EKkHT7w4c#(IH*lbLT zQyov&QAJ{IX6EQzqL!w9$Gz3HpN8I1Y0Cwf;EXJ9yaxTx--!MBWRwykG(cblvqfB~ zEPw;0$LyMs0VG(;0FwWFO36cpbge|<&nzCqmMe@GE`vMbbNcA0OC~c%EZTSeqRq1| zbD|FzOm}BIb+M(bxqWe^{gyC|GSj6W!ev<8TtM#e?6lH}1tEL+Jx0r(EosfAq=biM zJk0KP^RqMSh?*+j!oN&s1$Vp$%Y(QUX?O|33+6YO3p*<=W5+}b%6FiP*x=f7S zOD^sGczb0ioQbfG7S;*HCx+Q_!3#YINjF~J`ayN&7484k$EfkoR0!_Ft@%7W?&c8_ zGgV#lEwbI+i?74!utjyo<@0&1+1vwXIE+|*NdyADsr=y1o?01em9Q1xaPRZ^%P2yo z=lGj-k|U|N8dYdj($P76p~s;?!3;k$!0^>{+4hq?x>Eg_q`~|w0}EO`f3p33$;5yp zGsns#HjQ7vi-=0HhzGB^oyuYt!Rjd?M=iNtk^1#@B2-9e7B-Q2;E|04SazH_( zK>NEQ-BT!E)uv+GcGQ4BKXCDk6Ec3KlQMn<0{BREp?ZI1dC*|545APEy@Kf(7w|_o zs4PHhNsrn*qjm&mS6-*di0&z1v(={LzSvO%7-j<-b6mn;qvU2wgE(avpB@potJDGv zgB)E8!79xvmlRs6XHu{~ovW#zNAE$0jc?cbstN(MKArG*jQQn5VLk;}zUslgbJ$SX z_M`Xa%To6bICOl;UJK;0+tqA{E@2~FSF}SrEQ!tX*^~wknHi$t2;eX<>l~qlVLpIu zZDm(QO@?eO+2ll@@Woj8WB%oMA90vGb6rm&@(2sn0G!Iv$~KAG5fu7=aXp6N%A$bu z_8Nm-;IZtpU6Na4=I2=&!jENkne~^qiNA8fIeA&qTAq9hbzjNcnL&+s0!Sud{;(&N z5J={ZG`p|VWO&g_Hbx~1Uxc!l$i_#daMz`|haIVez-m89v!A3UL(E0qn0o-_Wsw5% z;Dz{7*lFXVx3JS?QTuLcG6c$GLxsJ>3-Ndr!er7&4K-aqOtV2HTfJoPbA%Jh=gMkP z6~GCNL~ij@*TM-0ezc#b?}v#$j;TontOsB;N93g&Ir|<(!}yepguzpSXmGaBq4kfH z?ZD1sAG~DolZK7BTx8X$b58fHJ zLsa?q@I{ya^=mfe-8dd2DYEP(OU$fmE*mlIPEGq(@2Js$ zi5xJ~b2p5(?EGW2We^ZK8zHg~uvEY9A8K!Y8x%KH1$>-KymEo|Fc|CC%m*4j%81$P z|JhJY=pxL5vvMLMS>Hh&nKH5V1JqX_!b7eOit>M-_1{PlyCAUrojuWeU}P<@J#vu$ zoLhz9qw9LeTw7$kYsNp#>_n!Vf-0K9WVXQ!SkT}9a@c1O49Imv2IO8{rP4_xHu#bw z$%{5Hu8sBPS+gL%b$J*ndV2!A+bZ~(jmbR9r+}Y#UnzndFf{t$H^x`N%+>KfXRdBc z>@A{9>;={_{F`*7En6)dlBk)#%ZgVkRKPb?djEooC zYd{$d80gRLd4vTKDZ*JIqhJeQ+${_U?uPRd$uEr$BTu9qa`oVa8}c|_5MvA-zfiQc z+8Bd#6W->_P+Mc+T+rXcoruhe;)a|XQ*XP>W~xx&VU;TrNLL&DHRT^&p`T&`t_ff@ zptT#S?D#`j@EYqI46Cuahra+?@W32+Bei*Z`F)KE9iia{n z7}O?Z<_r@ki$F69`TKP#&`eUZ-zh?g<;Gq7W8QA=985@e^6vD!wrCMyBrHUYo zrH-_=sDB8A%|-~z2OQF``v-^A!!#<}D!d6o2XHU^^IbLWzzneLG${iGsDbYWda(5?iHbh;Gs?}hEQ>nebhK>r$^!c8zB z3L*btWE8fCpGLTH7{mj}!0GKkSXT;RVWiAohB^$+0po{v`W3_G-SaXE2w^c4!h&X5 zYHt2Q*AE86F(cgrA*?GxScHu_|KJb|7gVk5^V>y{!PzkSi83PjJe*6#x*GTpR5SP8 zT!?qM!Ak!kKm5)g1Ga%p%Hdp;iOc^m`cN1DKQoZ}pA7ZJWNu_UF&56TA+a!PnnG9{ zNGzfKC6}KOTnHvoMiLQy3`O*zV~8@~`IqXGfzfhUE8F7|h{8>KVLKpyJu*sWaI5Tb zDtWl`h7E~D!#V!+z*gX&Z5U^HxNwT-!{!u;gGx?kjnFzSy1;>dh; zxK+wT>30|EwJ-{o;8Jzv0nv6XkT@qjMoI7XOGO`gRBi)+uyVBevM^Y)QO*0A*dcF z7xWCubwD++(C}Vb^BVEHx+MLLSJS3FIdLAQ?lq_4xbq0U$!yP}R+cn-x`;|QE3@+^ z#|y_w%3~MZl}W3_1}4>YLV3rr^uD>%U>IGQSv;qFdul~n?rD0j)J#PaRuY;Y%ayDT z`1_C#Gc&^L8GX`sj`ia!<2RBe(eLna!UN?qeoxo##i^A%YpHViarUeh8{W4c7Hc`rN9Mhi<%p4)F^(v z9REj+A_Cgr_VI;p=M?`2^_HeMhl|1W563b~eXrQVa}~&IT3qc5m7|9l4J_pf;qGVL zNqb|tTnVdjdLeU~=)yz1-rpj0#jMA^mJnyNZm%z+Bwl%al`Q8WkXG`T$BrJ~?q0Uj zgmuzA`FKm%Lh4l9x`z=y-DK$7P}@4c*W^yw_S=0h%w?U>c`6t!95?;NzZ{bIT+H&M zgFP^wg}Wf!uxoUo$FrqFdS%k>Mi4xt9opNX9~ICvb9D))f(NR?m|@EnE_ERRzKMFu!_ zv2CS3L49ja$vL8q^0Bk351cPP-Sd6dqy5ILC*HtoAv*eXdATpy(KgZbla~9J>t~!!_zcieG(A!QxRMNJjG$yqdOHoVRMjy}C|(U+I<6--oq>YEDL@(ZfJ zocNed9dBq^&)ZAm_i_KGiEZO6<_m&mDpcRZ4_h)x`74|4ykoc3bxq2vfsRx^94+s3 zVYJgCM0#nmtLTNS-O3BQ?qW6loc^RokCL4i#-$!`NGb_w#;Vo7u_+LlTjK?fEB>&&_RM07@{$T#DYa-BzZwZHet&`W-oyxvpJ zajDaN`kh;5U?Ecy6Sr%^_qU?o8xzruA5yloO)Ymyvq(igJFrAEP^u>gJ6loPYzlh8`7jpILRp-7ci4H)c zGu3{fO-0{4jKR~NXk33(MA*&7R8>9dLf$P7yxiodzbtZ##)V-*QU~jz*m&i$^#b&= z%ns_Ltjr8J9qEl;FsL!{aCw$#*Hx_*$k_Ju*@Ih#B8P{*8ee(wsB*l>ddU96_0Rh< zEBr1fZ=zP*Yqo1*M$FS_E0rK?FmRpUbC)k%)L%5<$|hHKd$Q7L!<8x5D-JSI4I)=& zKglXjM=ibirk>yp0VmqcPCM?{y*O+i%uP-80

)PI!U;9S?-3qZLjwQ~9X@PrrLn zU3}tWG3-8A&$lOBiHe`cr?T^zu>B>q$9rUqi``#_2`n4FaLh>Flk{P{ZT3=UzC(%? zUh{zJSE|tu+Rv_AXSSoOZLd?3Hum>|SdjuQ2a@Y>pNsCZ)dTxFje`A6Cpy;OXy?prH96CRHe%Q<`+OM=|F9l|<_ z5&K2YR<9W*_dXGLLX3~Ucm23&$%=SPLBZa|GqaM*Q?}cqpGmwp@0EYKz~-&G&c${i zymM*vv<#n8u`1tU70ZpK)Qhi=MWXwT8|2v>i&1jvujDvb@w$P>MYmmu?FTIj`8&_m zcotB9Ec8Fi9|Zlp%|&-d`NQ@*c4v4z85yYrf9~Eq{OtEKXvj07zKqPUycw2hzul%s zz6QBGN&wng&bLxrFmC933MYT5J3Fksf7gwD`E+r^N3;Sz}y?0MPbYEEPsV?UE zsSa$Wj?aUUOa;B-U0&~BMYvzzS#I(D&B3QvFLx7)+#b>}G0K(SQZY>l^ceceWEb_S zI=DO|R`gUqf!6ICm8`CTy(j64>Cog_v^`Fhi2nAdx8i=?`+IsQ_EWFFjF6^J&6R&Y zC>v;9GEKK=U{pE(lb|qSin(ZzXnEM;e56s`ejV!~x3ub|h6^#rLba<08w6BoUj)qW zk34PoVLX#Ll$7LO={%9;4C2zO(*BrO197|SNIFCMD)<|eR zQS3ggQ+iqdnS$Q(OnKw#TbY{#U(SJA<6zgdf>?IRFrx{b=I5OH77<gm^1uSZvX9nYsK&6z_O<$>R#|iHv%E+;yYI+((bhbmWbBW;3u#@dg5Nde#`N zYVP){#dkN+o!+w9`^tTqVa^@EI%=SdE|mp6Jh6#7!4LEncf9#oV4VX9qjBv%wrMl# z)M4PiI6w2Fz`2>McU(QT?me~{miNN4z*X}Dlx4f$%ZE4LjG}yvY-0Dcr{uXiP<)~MyS@XF6L<=ogk*B9Q@dL;kEr_UeRE>;d7=zu9E7?XjZ z+=Fx7U4}+}j)4QO^7_IW^?F%PC3ALGWOTm7^(FqNU7TWmu#{ObZ-i>(k_aEYlm6EY6K2zTxs2*zORfY3+#VY)>3It&@8t^?<;!59(aY+1EdxS(+_9sp;>3 zt0Fked3KNu|5kYVDLRoGpVD->USsAU&8TWP|GR?ft2?i9^7tfc@Vq(S5rgACqn|UV zOVg5~Mbmmd@oIVIJMEEe&X3cm!nyG;mI5A2_PytNMVK2=2o+VlaWP1!gVH(Y zS-s%$!Fl9*&_w6)*LniSd}J=axFI*gv5j|?_#m_qt4T<)QgTGCO;?Ms-8EU8tjfr> zZgQo&ukSJT44?YIt}R>V06S0bdguEvr8Sbe=WNDe|#A}$b`aoO6jM!I5X0N=@p{-mT{n`yb|#umBC~(eUsV;iti~H(u5FJKJa*8 z%8+wi!$&Dz(1Ch%%0#CI_y+vw6oTJWvPw}v7MTq9;?Mq_Xg;F%^+Ve_FSF_ zA6Y5bIX8hV5u2(|PD_1c5O$B4Z$1|(lJlLIo5nFb)l-t1nqd?pYrB86_0~P<)wPQK zz2b(xyR8$6sgVRxr^zXM_7tnPhFFYc`GC`!J4}R3LD*76^v3)R6>N)JPe&hAYwO>pa%1<6lI7 zUJPfqg7b>XJh0MUp2W<=Wp!?Ki0YHYP6>0pHduJ%;bS^Q3=1)HSiN<(B;rdJKa_yFA9Tlq@|_uQVbg zR-aZY!`H0;RI5R)D>kL7j)UeSMQ>6{Cq*Bv+OnB7SOmOXZTenM&|hd!R7;fQDH_mU zat9I+JB^ia5s(`R|nUd@fz z`6e^ytCnTmxd`(2X>+tzi(TK-TaFVi&(wCN=uorm?SJ>!RKH%1y=3;Qu=Yd=d9MHS z-qFV6#a0g5q@&bVT~4ObJv^b!_=JVOQ)ho;gz(yH^J?brb z;yzDQZ)Ip^_i?X-)$#DPY@>;QjN#|02iddS~4||RDa&b!~__M$C6u&g1VfEwP!**g*(QY1l;w5cT ztMlZ~&@U6Aas;iwTPZZ6D%L3A^>1PgSoFL~a!Ol8{3z>V3| zn9C=ztclNO>c_CxXN@(_z9^Aec_;RGQ?#-2@`CMaZX4bxTX~*x^(c%oJ{6yQ_(s_G z{P#U312o2aQ7qD6H z4xeTLo3#cm_Y7tECm(p?#M`GOk>$OxEMOf%4UBUfS$=<76ItF3%PcoSs3^GCcB*IhNa016AGc1lRw8>g2SpCv}@28^|IkOR-PF4xPeUN7J@5qWJ{2I zxtJ6mf7+&sF{#8$wrRRAXt*putJYZp&s% zNWI=_!?BAam)Q=uzB;-uvOw;3rRCD)O*V*!p}t8#Hz5k80u=qGxvB7N92W*W-e40T z=?D3YA zrY9B8MesL-VuM(SqKBiOn?xU(c%73Q2er;s zO5M695WOuScB+vlJW$eU@Kx&ftl}uOY7WWR<_jM4cshvx@*gznFgl9T_(PLX z-=~sWTeEs-4Y02F$tS78X{yfv8BK)lu!x#e-v5edRB##xXltO6LiuAFqB?&qp9YQk zzpQBg8`_#G_>M$kruMs)555YXUsc-C%*5AAvsDIPY_G|RtydL&@aZrpbm(*3ch55v z7r+puOLKYTY*ky4Th6G~c^|j7E5e3}RbB5+-=A@wXg|YEcHvQ)c~!AX7=O7g3LVv3 z6?!A(-3wXc{jvm&OD*%ilr;^@3-xmm`;+a3=Mo}>S+t@g%nztHcYbi4xMomfx&0Q~ zi8v*nG94+7#w3k8{CuQHi#AQ^Q6ur$2EMVIpPzglb=M}nV>ViOcV%R+=jy!MOXZZt4}|Zmu83d(Uf~QsW#z! z3_q~0a)&cE7uK6uY>Qltuwjf*OvxPg^f`RDQSVFhURCjddHy(8V)t1Fju7mZCbDlM zb8X3`p$J`@N2Tgp$Z~JLeY??qFgdf)c5b!!5{HQXjK&km9i+*k7q!FD;?U%oenK^O zA;2FcO17+d6ne~!E5q5c4Ie+dN(GE&v2xbSSs%SxrNQ_742r!j*mfXsIsby2-hIWcbkecqI^|B%l=+Q1fE;X z8Bx{3$g*G6p$#5`WswWcpznXuBFoBEtd!-h-^=i|#P6>Wf&Xh0sRszBa2_4=Y1h@U zVPar-E3C=ilo73W-`L{mF&9rrPIunSp9`4AzkF(vXrFNU+&l1Qo;v7-_{e#@0zdk@ zKdsxX3OPRQ7N|>X#*@Jn3uQ`#@F8XNJ^jx8gVsCVFG*mm`a&PzzF#_9qvg3mdi^qy zGuQ&tl4+bkzti(T`I`^-w+R$<$|=xjE@wacE2{4E6ZK?@mdA{kt~mWTT)7UA9K<$zN%yk)2fr`Dsa-`GQjglt{(z& zC%w^m&fYW`qagR=)!VY1PL?gS-dCmi)jPuYN?-HJGkkNE4O!1R(zzm0z1QxoTFDs! z{7ox)vBfv(tE@Nyfmw3I#CE+#$Ro$R4NX><96puuo$cM#@+r062w}}6l~C@~Wkuf= zL6hb8KfbLUA=j>ggZX}>lssgWVIBFUuUa$ygW0^~?1-Ff`vavj7EW@e83rtGiaj1W zL=s!Ajy8%lZX25zZ!kGa5}`rihd_m5Z<>}k=b$4W%;Eg+`0 zB0mSNh7!fok1dH}HXuG{PJZ@pG5^2fTR>m_xf%?@{~v=;XA;lw&hD9#kkxUIB`xQ| zEF=%qTXxIv?4=o=ruGvlW!pi43|GsjJhMT$2~Pkk6b^idyg8vdkV9E1JNrV8Hje*L z44aQtOY%c$2JLM%8wN_mCQG)JKsRSo$@KlK}MNn2y{#VMOr8oQLBNr*>clX zyHXgY%CavcR=|&z7&xIXkCkH2@I3wa7L`$?Gn)?%om450lB5#Lna*Z)BPlWH0cf88^p?NVV3T{W>C3ns zq_CyG>`m3AiCO;|k99HiX`8m8LgS}Bs3liQ<2}<0%n}(K0A9z{G9iy|DZo*JjSKd! zeA7r?Snt4wi9H9);PC{rA62Gjs;2%29dw!y;&O$W@Vhz^SDLX>F^DuOML?Z(pLbqxcC%v2Y?ou0G;?({0@K?N-r?{K->rD zGv6RuX5!WFfjHO+BTBrQDE^Bkl^E5&2ZQ`023Obpd-XdY{*OW0Uy#&BG=94!04s}N z;-WZz{*ErEk_6|4u^&I47P1}0w&0nZ2hhJ%kU5&^kEQMR8t4zF0qV#gYytDTcu`I_ zP6d@*Q27L5v?!&X9KQ6_^JZTx=2>JIMuW#dWv2(A=I97R!L}*VBUv%lgw7T%ckS*E z&Ys5XzCFT?oW4h-@dsIINkO~^oKIcjLfcGv;{9(qtZj9Gm~nhWBVE?Un$z`zO{-9P zI78Xy*u&V7XLGk*D=hik#90^~Bg?wGlFRO#xhSRYKU>~5Ca-O@+&egZ;>fO=cgV3TE}Hj8zwI|(nmgn|BP;z4H{Ty>^7!Pa!b4eP?`QM) zb$L_Ily1cPXoc^WdcA3q+WuhJsi%cC6w8$rfOwkqEOsRLUVEk6GxTh1cQ7Nf@;(vA z_ajD(e%TAxfNN3(T$9GxunFKZ)&s|L_8;*c2n`!8!><7~sRtzJtWny+RdBT&5T8ti zKO>5jfLJDe;nLq?gMY=Uz`yzDYA}ewe+a* ziOBZK)TW68v9t}}j%26ch^D!fWoOfp5$!?i{d$hx25@ZMMRb;zyUgCi?H~3vQnV=@ zw`cesgb6d(v-Rt6y7t5?=UHaUie^fC$C3xzW{#QAe|zzAmpBP^tlOa-9AS2O^7 zp&=*2J&8f!dlAWf_z_$o2<|Js0T!@Z5ioj{;q1m7RPQn{LuA_bEXhqTAECGJ7WOht zE|O+F+ya;=Il{_~@ty`_{Da+A1$W(Y*cTE!#cS@ zM5*3=A`UB3k{GXf>C3{hc73muCRObF>g<96MZ@OAztW{&;x2H>e_C)+e-PRq z(;l@jRx^_=7IpF4IXTYRuGF0nF*YhPQWE((685M>4l>f~U{Y)dCfD2|zL8Ao-1Fba91QF2zqW$Kut0n{35F=QcjBBz%7ygLd{uL(^0d0x1^elfKxe)JQ zUJd5S2E|-X*Xb& z5rf_A4pd}zK+C@L%2`m7SO6bJ$ZU{G7%V~g-%DTz?y%^hpFV$<>Xn1ZRq!<>MuXwP zpFiqmqZ%d-mm1K`LfQ>95z&^+@hyidIP0yt-8$iqg)XD4ZRl3)Jf&%1`uE@9cAAgG- z{uKlA2qwaHruok!&y4p+kkf&AJ_ZbwE>Vp9a3;JlTaD>2;x@jz=>KA#wd~^AGN6%3 zqOR|lNJ$a^gXQEJmGmc@r0Sn+60k=!!w2mv@gZ<4ea!pvsITw9L5qq8aT{6d<6da28_&b z`x@CfpeXiW$f{?b*n#@U12rZ8K=u}Ri66txl&jJShyaDScLsbxXFt560(Qv_Fm8*A zL07<^8N>vFWa?K_;B)|DhkwN64}iUUb(@V8JjMvTNv+c-2pIY-ap?D?Z(m4~+#@6l zAjp7@o}u*pe}5NXT+gh2V+6W;j|t3UL@3A;Jn${hmu2E7^S`ezAHy3hPiv{ionQEZ zDN;EhAM9S|zWd(IcHrx5sQQ8F;<&}zAh6JF9Pg{lMI{>pA~s1LJq7}{UqtOWOMjEN zCVXHnCfaYlfQ3i_iz)H?jW)O>h^Wx!`cy=E=L+z?>0UW8f=N1obqG?uL;~O$Ct$c{ zNvJf4%J=~&_Iqpjr{G2nuskVWzH);ZJpy<>gAhMi=P%;N<@pyC3t*Pa!Kx`*{KY|> z7Z*5-+WM~D0;omY+IlH70GCj#33ev^ zN+d8$^gso>s%zxU!Hv5>X;xgnJ_C<71G~#0CHIbq77GDwHs`z#;?5V~gXK4yzldRo z0gBz$eI|Z!BduHW`O%hyu2+YQA&9A%Q(avC9!jVjmK`i}( zcrwV+-2_JIo)~zOlI=GO;Ko}-=J=mY;RLJy(w{070!};I?s_&JFc=ruKzEXhIp9)q zB8EqfWF#d<=Krlm|E)&C4whOU7T8ow|3QIM1}A?3-3! zDraa_4CUHHTy~rNCdw%1nce&nGW)h_4q5l$rOn``w)e@Agx`Sc_3^sxF9I2c2c^$K zbvhPsaY(nRZbY5E(+?ToC`r!8vfX9Jnrs)=OX7Cd#vWhXbAaVsj7sgBtIiTrr(7G= zaLtJ*2HT>ZZX$danvB0eORMObO`aJ*vEuUcXXZ@f+T{309-V`CF7vz(8H5nC?HYWw z-UlS9GCQpHdJR9v1=L1*?;iPG74!<#!7I*E-qk$Gh~@KA@g|5Q3ZzLX1guYf1)bFy zWO??&eD!h-a^f}D=~O=oynM{s@^SCii4m^KwXAs%D@zC$LiBWSHX_^zElpyp#zlG& zmua~b4n264(debF7J$$FqcqDUCI2=}Da}{k|cgd*1 zEmD!($};GEYTJUX-?say23I>*L&-_%F^6hdrQheLB|_yhQKIE`Uv+>ojCw7+3Bc4H z`a0i;llGYKZ;B+y*xp_a_ib}2NA$*r30d@N^fjjqG#Vk9MVmLzZ~*-vAwQoDg4Y+f zfKy9-=gKGGLEQzL%SZI`{~tq<|Jr^3XrOnF?PEIe%S$S={J86vjDEgnE6xa9%7b;e z!Wba}#iI0t$?S~EdV|LLV7_vKpdrfn+&O?Gpg^8hxnvvzw*G$4RV%_ z7y7<*zV{*b!0nvj5;IJi{YC59>8l#0f!c*+;I@?a^y@9Y9Pqz>Ss7Hhd}{reEbzzz zVHj8G&RF@(LKkwjFG+F?Gb`3^;KrzY{&GSVz#17PikX?d^p2)PEir-bOT$-Pe=mbX zK}9R&zVp6pWTwD1GgrL@;o6rwxBU9DtqT77EOnfe8*y#_Fc~&|yW_Kcl}W6m{e@m< z!>(hhHe13-85OS5TJ*Sd#fL^wXN~TlSgYpY^YjV9wbIV$x-%6^(vQcB%!1y+G|D|z ziyaf5*6&}kVu3lv37#pQklbTdWhncdb+w5v+dYrgAbso5|LN0PjAG6qj2_+Zi-s3` z3|E!z!2Q$TUb=7uuZUWF6U8(@BS!EaD2ljr>m^G@(N53Mu~a&9^xtOUpmgR8Vt2Cf z07h^=x$%G13+Vc%0dDo$@bZ_fBsqQYBG$rM$U?aG!;{kFr~bNoYfmSiCERCuL7K9t z`|8sdH{idk0qPQwVa~1t6avKCfq_IQ1e8Ds{6`4{)jeWHCPP^eKyZji$T?s#EdCP- z!BfA&z~qFrjW>J3jd>g_gRJQNc5=kzpZYHCs2f~Xx$!v6vM{UZ<%!wAlbu>e?U7K_ z@O|1*)0?9^?Qs0Sk*L_}h9WC~OH#Lajym)33U!TW%(v%e(73X9uoO9;zj#rBOfD$N zf=W@Vi9nS`LOgnRxez0M*W{K8QkFU(4qJ_k6;xwU+4kObB6g7}1mV^Q zri8|H!d2$sx}NOAwKts3fy;JwE&44%`GI2dN5^f^(Ske&TYERV?-{EJ6A1c~zT-`K zqB9XY>np6!yB+X57g%8uWG&lkvhbCX4%p{vZ_Z%nz=^X)jJfFTU0g>AMT4T&Zps1H zGT*Wl>@M^zL3zW_r zH=rCYphrSfB|GEfnC@sst3ktM4kn-1dd$CWe%JZ{)Gmm1e&EO?%oybg`4EP^y!U@a z&oWxl_-SWrXt>DkcW6Vcu`Zaf)cQadt=YY6ki3G zXQjWN{5)8OIbsg9>7E#`+OJji#q$bx?DspSaL&^~(o5647koU9>}xbeG505H{euNko6r>V^EKvuF>x)Do`DR^ zV}6fq9lUxB?r|ZyQlB@-d2HQcJ{2#O7l1X|#om4Lu<(Ny?{(Xbbf?Nl__AT-Ja2c+ z^jJr7{nxQa>47#mi|NOwxqg~+wu`@8HIl@QKj9)Ka+D}p2ZoF|oo3q;W|U*_N&Q;2 z;knS|(DiujWd_Abx{ZttlFTAk&q0tmAtPNiV2H-Z&Y#|LxSH&dtyKgCsbxK>O=Nd8 zQo85*fsN&DwrvSb@BggVF$5iD8WKf!o#_>r%7zQ&W;I`avVty92 zFK4Bhk1d;$qug0>{=nVQTUL468EgI1b?aMA%vie#64LrMN#GH!zr(g`n0n_maY)Ua zpw3?o6V(st`U2%H%1j9!hq8+q^w!6UJijbj#cMb+!g7xPg8fRq#MbA;J4*WG`b?|X zgo6U(ItVmhU%Q}r(*OAN@200>oi6KS`mi8C=+ivT2&O-XRy7KHLC?T9q z@GNYKFL!Wf)nLf_V0zsSr&pv&%n{7jWYBRu(*M|@ot5#hwbKdPu6A_TuOPWO$^L=o zed%);9*YAt5C^3W>l6Xj>v=oeYBv_IZO;)QuBox;?v!( z5rTPKx-QCLyxw9w6$d_xIKj5>Np&B+jd2641?N0w_F|)=@$$&B2^egzh^x?5&a?9q6Vi_HkXlNks)dc>xtpdB<+c`_I}3SKq*vbCMpcmG7d2(C zWBj%z(H%iOQ?|o-VTpUPTRT`}CXXKJXjrFUdLxoL!IkqUJ(#~~b7@vW@>a+W8RV(oI` z>zSfmV$odN6{LHnISNA;ZU_CA8KuDGpD#!)j$&3Fy!R}#vLf#JO+|P_q}{kU?l6#nKD1|iFt!=px_Um*<7D#dvkG}ng+ zhPTr!G+8Vl8B42h0(vr^`pJ3F!^V=ELpN!Nx2&+uvd4mdI(T4_`m{e5G6ju9HKeJV z`W`k&j;Yh{VWOH6e|?;FfpCYt>=KqA4N^DEcxkE0sM4N~3q7dr+hAB48)i=(`R*gX zDsDZlo#azwRHJO_A4&&LV=Xz5pz1u9&4kMJeo8eMvK`Mmc%tSt{FE9tx1Vv#^)6Ec ztDLWLSq?dB+fw86uIbZ4-5FtoA@g8PT*M_kMjfwHYP8|1S*^z*?bYQ`6{DGJr`gQ8 z%A2KvY1>oTju2V7s}#D>7XijM9KPtB+lVT4)J5&ti2bf zrZV~D11HS)cbKqw#QBSI*@43CxO`swq6TCWPZY@h@gdFtA4fc1kTIAW{8U0$x7S{-|XYnmXB-5-ea=P9*oTri31&@rQ zC$F)}{;J7&rk-Jg$2P++;IRb0y+JAb6qhUD)FlsyJZe|UB_4|TBJ)TLYPM()$ zCL}Z1YOaO7e=M^nf$%G&v7ysqXsH6JSN>e~;&(p|`2I&R|0Hq0+sqQ>3HaS2kK^Tr zrhMvOk;c}m2z5Lw1S!{dJ1bW^N9&U`gY|hiIEOgQRU^j#f0J~EfG7OYBV$29VnGb(|F)C> zw1oZ5W`UT`;Xi7leH6dgXBH(|)=^mJDqiv;g!7I~=welxPlt~snBpBuqj?L2F1)ke zy$Q>riE*6SO?MJ4CATZTRp{v%>fjWITd6544iTXal)n0D_R0`uVpWj0b=k0rLcPEU1R$GIi!-Hvti|{ z+N<^`p0&tR2FQe};@hNIUgsTweoDId~ z!=CN_#+2PipN|gC>kDNdbj-X+mNaF7iBJ^1i{n$czVx%HMyR2)gyUk=HY;Dd$+n_* zr}n*Ip`Veg(lJ^3g?8SfL-zWq(`5(u->J9vr`xqhrC2=bQ!Ojfff|36c(0BuqBleI zQm)Ho@zOaKONeI*k#}v1%Gv8&uy;RuN;Tu$(>DR-%knPNq4Gak2r<1AuppQ{n>0h4 zHz!oowz~9LTJ|ahdk4zy!|48{Z`(5SqpW;!uSfBA`Ff`%4qV5C8AYt_;ip;6eaA!l zsF#VgQem7Kz)I`O!K1H&Cr9L^=acx93sIM%OgtwC-FCLTKtmRl5aZA(X!loBZ<_2&)OY4W zrzowq7AE3tsY<*&h+g_Y~e zoYWPpXj$xyoH#^`_BJh~R!zEiH~sz|m(EzR^h7Iqt79e@F9_3V2-vGcho=g;>sQEm zxv|!|ET&w3W+2DTt_h_mG?eVJonuGz#c4JKfJ{kF^7ZWQNiCYn3$V}e{Pu+KScoe{ zg`?3k1B#o6duQzDMFo%+_~W?s^qFhuhhe!8S=cAzDw}HsMkP-xhRS4TNn|!v4~KYl zGNYEnW+scSZ_t&DebTo35r25wQQm7XIG{vP2UI>CsH!#Ur#tGNyt2VMvOoG<6PhdT z1&Y7Mn2vs5I&Iu}rGO5GclEHhGKJ~m_M3oon1Vdvi19s=V-0RT8B^nh*#l> z?AlgA$ItQV8TPDfwF)1hloA%BuJY<&JsXafjZVGS1#OSk7UBkt)#^-vb6J`w?Ije) zZTyP1moYE3vfgI1OPKfN%Kdu(o6Av#S>pZ$0UmgDf_R3kvf-M8_ZnO?ZQYxYkhrt4 z_xs>_blBLbJ5$BXsa+Fxi#g($s#rKC7nA?K(&H+V85nJafD zAr$SBe^@-&NmWY-p0w^F7@>I_0V0yO5ih8xbaC(Ivo?TQ_;*5$w&^-rX>Mv_x=W@M zpF*r&GX6-3;C5bE>9}$`TeCnV!%%y3+{Nw1o=Mm>kqnHuJMM2fJE_AQ2 z#s#V+={jAHx>ZOoiz3-@7n5O?ar;Z%)}%W)cC#ERo@+?2nPlT(hsz1-go}uaBDQnv zv>5iktZ`@!YcVLbituMYhbV<5>5Db2qP`W!c6P^#GFSZGL@?o24a6uKLO@InCoI$)DAQ37t%k+yaA>i>njf51_Xx9OXNI z@g1@ToUlAO|036cuSTZtg55}|3)UENMs?>!Zg`!YLv@WZs60@c&~n#TyQJl{ztRzg z?w<`#*EK0y=e6NiV}QU5xI1#WwxMU6_BSbDj)mQ}3rnbF5r*Y`|Io{K+wEW1lm)NGw5R}5h99J3qT0IRubt>UaQnNe1 z)Dm)P>{R%B6n~g@hD((ONO^Dljvh=E+E`qK3;Ya!St>~LhOURD2)(bOppu)<&pG`r zP9Ieuh199a*Fy!NAMp0wf~38b_EVYfy_wmw3@c6&Hyo&Flolm~F1lk@oIC4z{Qyr9 zrSHc0nJ4~acbW3`n0ZWSljz9Zy7|6{&RDL#Cy?WK&sBaxmI>32!d0z?JpwjNg_22R z{ox(BU)Ur(<5NSk`kVf!{1^*+pDuw2`X7-^!|Y@C(c(=XV7Y}neKAEoeoyv>V|gU` zvcLMChkrs272OpsWqcRl#wrq$wUti)fh%&QHBN8c^>A3HQ3&Qa$J4aMca6g5bS%qg zy6h&)y%#_%xaG>%5*0GW76n60sRcHRgsIGt^s#vp#?g zQwi9iTuj6xq2!*X`tZ}P>ojo|En0oQ;;vYawqenJv|__m7953ka{MWZ2I40f_qkq} z7246DeU;b!@kwYtewyq8amZ>&iG~;`P2G#KXdA+Xe$^GT%`8gW;T~EFf#dcF-i*jt zVa&Lcxb@cOi6hj#s(o0hp5ZfsC?S6Y*8dnvAr~-LjjKAD(rCnQC7k%77NQaaY&1m<|0|)sgJN zwo2y_TYij3k0Yot3%pnd+fpy-cj}E=5=J~Oqdse&!^ZdjHjrB@9|Sqy&vy8}dENzG~`7fj-cBesHJ! z_u+g3;n?d`do6qNKyZ-?-Q{CX0!x!A3eQafxHPTjo+V0TqkG> zsD2P_VyeFy3#l5&3Ps?4vO7_c=@D(Bx(p`;NjA~zv|pmSPev*NY~q6))|iWDwTU)S z$es*zFG&VA@#8>LhmZ0Xj?DGqHR{FQ0-v7&c3g;)BZux=k|*odOT(vEu9wXQ#ThJC zK@FgY@o#swxH%Rx)^&08kF!Ga=si$`We|tz?9ZOUf@Jeyfu~Q!V=s8HXlm}K>~Lb8 z9}VZD7~lmFjgt{ZkF~yvU4r$-$@!+;GmjS{wY4uej6U`>bQfoVMaS{K&Fcy(-V~nB zW{_|FE@ATB?z-z|yRil#d}!{Fq`w50g)ziDOF1dLbhft}DyXf!=03+S`ik}8q(Suvym6)ub)2!n|`OVAo??;tr^9rwPZV%~o ze=)rskN#5B_&lZ1sPReWp}O|?vM7P}ZhUZZ`Qc-u(Xv@5)l-@Ye${n9!=xSy!HtNZz?j9zXW0fqzgH;h;rzb5*2Jk{VB3|gBICipcABfVoY;pAt ze}Jp-+M+{=83m)OcEcN89-<%lPh=p4W@N5XSPYmN-pL*q`My6n)l^=%(pWptWKm9i zoZ5zcq3IH*3E}m&joBAA@6|Z<*)!uYJc1%>Z%~&cvQjb+Ke=MrnD4fbi}9diGKhD9 z8ZE-aAoKwvCTC^^byBjfZ{au3@ef=Rj-ZDCwChn)*=z>h(BwIqkB(1Yr6Q@F^UHBv z)n~t4Wxnk4`M&1rD0y15Ejq*2vU|vCjU$hD*-n9aCYFw_t?;}rTtmink3#yNb(K%XP7@-`~vDUdMDw(_wzk=CA=h{ucG`Maz1IQTvvo9eK4-7%$EC1+RqYRVwa4dny7*UaXQ5etLb2p1t#Y04 zcy(dZ)zO_`A;rLr-I{Fe;WOv&4EcWwYGpa6Nv9Bd#?#+~D(cn2G{u6oYsGFVF;b4cpPQ zc5nw}`N*K^H8tSC9JQ5>Wu?6dE~1vXI6g&#hD`uGgyktD%qPc-=hNYoZG@1UcZJT3K#@yWNq z^SXqh)FES|og!d${FZ8(%58r`i;SOCSk^QN&+=;KP!z%xEtXHrCxPKvocv6nicTuq z^InGovjZHuDMuMW+tmGRIMcE=#U9H0WfZ&i!CugP5eF+eNuQ-AWqSj+o5}}Q52t_R5lw*t@Yx10R0lMMe4yLo|Zb><_i=gGOz;GZ697gaupK-8Hb zQ9Y3PT)*_}+u2$W-_Vh&z0bRV8on-O@;<^^=V3(ioCc(|v*u(~Y4afG=SKuT;$4I9 z6eCRexOX-@|7yNzxO(E0r(*k~zWJKr=$IotPT!1S=2?>JNd&BdlXS z%FR9+b36&`@vehXT1K8<_IpDE=~F%IOGJ0$Oky1E3-@`k2^Za{gtxT&ZSip`UB_)N zb=npSm|kK8RRb*FXx}1C=V{iUoxMZ#BK8sf97WqDWHQ=8cW@C%Ckl_)T=ad4OUL^FUDhD_`w`3SdW#*rF<5M+|k~ zVd(SDT;VlUVM1uh?hm+BLEW#;P>#5-(is?7AK@o;la8dr7&Kx>+Ee2&S@g*{7ub3^Ye zAg<)t+-GjIS>_DSJLxQY-2Jh1yvF^_g_pL|8TXzd9;r$QuPMXFh>lT zi(e+}V>w{{?vo+xr#T7(*2`B`tHhOH$IH)8^-sOm1Xmfouz4_CMej*;>p!OBqcf=J z4mnxyYV_%EVmTSy9&nrPM7jPh#OyUBU(M}2On+}E?DD|pB%J01#iic_u-p`$25nWp z#*^THAt^6B)v44&aG*SK;Mw*@VC@d38I}&LiU{R{l*Z@Yyb>P2Oiq$Wtb+oqKpipF z`hQ{t{`3Jf`lk;da9=KujhW4{z0d`D5@i+?&VTv#l zYawrANsqXmM&XKVmWD==zZAs?`Y_hD0dLaw)0EQrj)xKYLly%9FnwP_IK_&rTF!}7 zS6pC`v%-jq&PH6lm?HcVj{uA|vM(-BHJ)Fk=8AbvzJ%~ItwI$(EF~%mD|h5z7K$A; zZY!SqorV$L@}10m_L`@ReIyzKdqH_#sL(FPV3Kg7|TDU=;r%9GC@;a>bc>0{OQRvr!S@f6=4Wykk@HHt6!a9gz(Yd4>_ zr^TSf^SE=jUzvJjVYFP`kw-s+FIB=TwvBQbjf3Uu$4nf5mPJO+ls(&}U7bQ_TPDt8 zyXJ>IB|>MOn1HT3VVxz<_o>CMhB@PH#mGaS9i%Sp#A045M+wF{KXrGQ50*M27O8V< zp|Z6Qyt$!e&ujXN>Cd#@z%wcOBAP$qmUw_Fo7K4C_w#H8t-N+6~#WPX<}y{Vl7lO*#t3> zKNTd!EigRv1vl=3(#Dl%Um+k3h6ALBGe}6iBi6>TfTR~0p8FR_dN1w&TrgMh~J@grh{?n;aOU(V$*ZoWCSlHPN&*aR&2BMQ=%F8545RX z+)^cVu%9NtowcA>5wchHu^G4SY$*v5Y|KJm!_;!M7WK%RMUx(F#XecQ%@mjurTdJpI9$Yl_}y8nh0;^28bqR! zg|}nJx&})P9(5*(^QOwajLeQC7jN&_H0S>CH0Ny3h~9@EZ412f1SgusEJFqT@OCW- zVKL@KEgv>|gxr^~FVU-%D58zz+K#(|l5{S*wi=u3iwU2#Tadjz@hL3*9-;P3){F@7 z4onm}77i6G#cXwN=LU&+={U}xYq-DM`Tcf}5TZlgzLo+SRBQ_qc&`)h(+jsM_G)&~ z-VC=>03JfX|^Frhzf8KSiG>jE9%2wuV z*DEpAo^Ld(Nj4N<;#gZ)s^+y0x zcqqqDeo7Kvc+BM#x0T|yGI5bgh*))*t!-vYns6x<{Ny|CapMPu3OslA1*TF9*J*{`96l^E5To`qFKsd8`% zYWw=nsU+&tnA_X+ER;pe*R8br&ZY=%kJfe##a2Cle7M{q+{D~%NNF?m83rkxX!1~E zII`2&Kwgj}MkCYR3%ulq8Auut`2t7v~?{wLde3X#FjtyLM zS+YORIJ0PCCztDlZ^U(5MqHznz9h8RKoL%Nu=TcybY_tdPAc{7Y=a{F;Nh8uWhLPu zc$0UCBITMD>|8-}Ke?~V%EHvKisPhz5e3sz+q>K2l-r!#%|$z+YioxFthWVVgm`{S z*`*(5q<|m~a~QAAMpsWZxW+V9{}Nd~l!kWbHLkQ{=FkB$%kZ`z0K#ZT!HUY~XClV~ z25LBvtA$tpH3;q$DXTtcxi#5InMF8dz;aE^p)aP8`)T!Uxh_JUpfBSOREg2HZy*or zv(Q-Ua#g+}24Tybqd?Wqd&nU{SQBLpp`pzM^3s&=?9q%=Q1rkUy$Ocf%p^E_} zkWav8&oL1C*no~;cu+iIq==Y%DDU-fE{9ni(=Z5GVS&kB!5u+F))Qm$ju%Jw20R!$q*`c1x9 zV3FemxO(W0sCSixf*4=xvrStIQwTjBuhDx4!v=-S>RTm8Q%#8vCks!qg&<*@Zl~$N zRvr0Jbh`M2z$3)OU5&abmZuhG{AVCXFBykZBAjvov>Ybl%e$y&Z|l4!5203!V57KT+4sno3l}K zUg%Rlv^x0a&XY!f4}OGo-k4re(_B1!BG(@h(DyWcg&|jCW{+EjsTPq6imZF$1eyFs zW`*5X7yJhnK}rG7MN@wtf)0;dgUEpjhvppbP)BV6wwUqs8i9jx5Fp8wZ#zvAn8F(c zuGNf#dZPKy8K(H-g&2#@;yZEh;5};;4=tz02Q`N3m82JUuEP{Lt+#2IOi`W709Caj zhuIt#YfyvaJ|4rGBkrD5*}qOCHcGb&%Bv>E7AL7QQ*}~ScXmWny0Legs~Kk<^Ah^K z#pd-Lh4wTWl(eU_Iu;h>4xw>1Ve^`~my7Ro)<1Etepr&I-)dNDYQ3LugPdL>xal*y zXu!1=YpBsa@Mdq@+xinwLO7LEfUQ(H7@}C|)KCGGtI%WB$vg18kL7EJ)`sftePT4Y zFV7~+Ugm&afp(~x0maW|xmr2e$Es@zOepkRp?ziCkBYHSc@Y*^)0L}NQ1au+Jk)th zEng{NM*!sgHi!HPaDA2m#P^5e7uC(R9d8njVPZE!AlX$-)V1Ys# z9_WSt(y9TN8N_|hKbn%aH@vaG$)YVrww15(^$@CZ*5r}P*ITIJo2Oe{+R~z?AvMp{ zdbjF2Ge%~t`ywYROq2&3ek+Lqqw~ijDUH7nm73Fd??JhFXJVs@w)t@Yt~|5*Ua zIcZp6WOYr9AV5IKH3uzQhF2PfZ1FOF)9~@-9N*Tx>V9wDojQWESPkGIBiEc!P9vf5 zcXN0G97oIbp+|9M$(X&~RVI-PxB=)_!rrP$^|(Na@)=ePRlAxM-JtB&7t;eLd~BUl znVdM-(DvSICftTUWrt=R@Lt_dyQB^O&4GQCn&gWh{}a9Rr%xQJ7GlIa2Ac*42?zJ> z32vQ9+|RS-Gs}0}kGu)uXjv<@aj%`n0a}_f z!RCkv)3>0WPj-bjnu{wiRRM<>bc5UNCyhCSGXryt3+qC!9#rPE^iQa$vy9GSqsSd z#IkZiKsId%XQmY{2>-?&i^aOCb?q&kj=Lt`NJTRCZnSD6i;6%PY+dy{*xL{* zo;uBa!Di(8J=Knp+3g`6=-i>8XFQ+5+a%iw1<_y>)7IM)A#U^F|k=ytS*`N@@9GoQ@JK&eSobi35Of?|;nW zAF_yP-jI7hC%m=)b!zTG0kg3YNSq#$-~fPNV&RIt(#b^ET;oP(;@~`ou}@4yRPM-Q zuIw&gElM10MgWp>o%(rA)dV#8zP(|XDMcQ}tzMQC>1(mU!PckOIb)f`=^LPA1z z8vQDf&h-oK`;~X_sbv&iS5>^|nS~qjC%vQg*nXHiDNw!h6G$SyhgAAD%*KmM)G%lH z=GnQ>=(hOH+WlyNlX(HV<(Oyrm0Ikpj1TQMM!J^1NYYL`Ps9pfy`*W6eutoc@<_(t zduDn<^41f-cfu^V+k<8W_n;#qcp7wx-E*cx+<_})=Xbi^!Ixqv+pz5P3T>5VgyJ1R(z zA~rm7L4`f`mQ^tT&R10c2Z+XgrotNbiUxD){y?|0MXew*Q^0ktKRZ_H)y1*Hjc>tC zGc3D06(@@b;Zu{*fAY?E3_~j+iDyon?_c1e(*{148mNVT6BCvT+Drf(CG=BBYdL5$ z0g97bH%QIA*{IGDeXJK+kULcGrT~ouk^Y**u5fIPwfh+De*S~$TbAAf$K*6 zWq=?~+JP_4Z`SIjM$EO&r z^LG(8kfZOuX#?BE^KKt_7gg$9a|HA`xfOe-i)iOU>rvcB*&^s~X(_Sj{G6!t9FTX2 zX)S}rI)BG4j24YsiTxrqAUsLu=C81LWbyPH8*XoYvfOP~Fy6IPyTOt1a%I4c6z4qoAc|rzG{rL#$L~sXHpqyK5re) zE-rkc21rZS(q9Uyv>z`N*kDo(3V8{k&ir1TSvoUi9(OG#>{5`VXdlRCnyh~$NIJ+N z|E}*|UhO=i)pKE8D|ZOKb>m!-&I8&^&&{FB1K$|B7JsdApjj7-+Bd{H_4qG-^NcJV zG>>uTIu4zJ@!Vq_bA2nFB5J^I^L+ zM@snPGm4YS%61fbNH04Ly@mtVE=he!j|^<|#`t~TNi3oKJdL z@XbdZID}YHFJXu=G)a#~du_BLX|50a{LV`P4?Oz^02QvsyrUnR3lLAvo5L}xi8R}z z(4x8bnWnUIdN#BsYo?b1%eWSmA`Q^6PFEGN+{)1`s3sADMI0@|1Ti?8E1&fvF66@( ztse}vK`Vk)5BABFbF z8OolWe7mzycj?W-#-vZVD){LD1@g*LS#zXVWz!P8!Fj8hAz)FPXOEa3Npsgwi$NeR zXxkZ#TV-+Ji?}gF-f!C0oyNI1o-l-{VXe28_1Xop?05rrxFJIU2g7gqX<6&xn%p4?NdEw;0-yP7$65bAXPmqfYq zs!*4+J(ox0H=QPHt_n^Bf5%al=`=vTtnj}g(MbTt;wj)dk;*axrEiqUk3n@_CIFZF zDo7VttsCoCRoUur8O?{)*~uZ8 z#gH((8|k!=;Gy{Djs-5P=iZ?(2rHC)P=G!5d&@&T?&1B&sE&zhTnl>}TGbdxy%H>*3zhzx%A_=X~M!~zY$5$P|7W5ic%q;vevrgM9h zFF0e0&yFXGS%sv4#5R7)O;8IIglMF{?9ruP@t=rV{~?lp2AlZKd?P6-0T6ZcSfoe< zPK7{E1CNiqU6;^f<~a(q3}`}%C>srQTD}dSEEoDZCbMfLj+9Lx%u-F<>qQ7`-uW#~ z>%;_ig_fL*Tg?9o;@jvR%CZV6nE4cW2+fV*WDkA)wYJ7EeBc?IQ8gu z|Mb;c!grWazm2WC3F2o{WX>`=dYE;HJ|XD1vUB;B0A1obQekmGzeUw)$t`?-wh6Y> ze6iwrXp%Zgl@A^*Vm*$-n!%rIUz6 z388?%UWvxGc8-s<_42m5bGd_u!~5Fx^j)D@>-2)O97Itn*DWve!z0tO@nZO~d4F0V zuVvjEe}~hWR$nNX-M~t|SN@P#_+`ufx9$7UiD>;mgTag<^QDztlbhy^7S%g6-Y5-c_9kL!fc01sA z>Km~`mzltTB7}87^$CQ-ZTj5v^qk7@h}gTXc0U_Z3-|v6XCR&Rv@uzg!eo{Yc%oRp{FfIR5dyLns zs=s4!Sh#0dFW_pKrde)*UqS8Mn@qY6v&!8$#s3`f-_QnZw7-D_-f@Vmo7`F$c9i$R zx~z{L!7=uy4ZrG=VT%^o^w`G;rx^!m^B}|1i<*d2l+>!kj%Am96RXX=hRVt7Np$TYFbZD>6l^aVcKdt|wty38pq zzP4|^b739Lxqs>FR(emff6rlmr7#26ul!~|0fiPt!>}OPGHJ8Kr0@pz#U6D_ZYG}uxaOtZ1Thm?ud6dI5Rth)k?uf~6Y^}OQ10z%fTSqW-wLmKMr~9Z+5k^m zt#9wBaZnKa*swDD@etEcSB=xA=!sE(%RD=dP_X9-2_QneP9kFHe~x&9)7kWAJWb_{ zF0N%vFqUPrYxgvpVm;EAygM1Y&zcm*JR&;^Z@4x~Ju&yoJ%&PQ&bo|`oikg79T36^ z1nY$t44XRSXC`e_`@QR8lI{1&3`h$;2@H$&3!UA~S9MIA7FW%CY+uN&;8(2g8#`6Y zNwiz_%BZ)D^cSVyEM!EpPncSKd|0`t+&{&KcrA!t{?*10vAJ1FqVk(3Lgw&rJy<$< zH+b?ARVF|h#`W%-=KfV?d!S#@=!Undg!0cC{;ny1gR@G~%Faeol+o(MBDgfn+CzL# zxL3(Js$#LsQ~V=eXe{Tp@r_MBiMPb#NSe{viE9y<{qH&u^)Y>wNipGC0h;{dZ@WDl zQ4aEVbrIXcN8U6P;v)y8ca4}mX3yT6TAMLpHApPC|M3^}DjI_tW<9(EJF=gDJ@Gz; z{f$>o8PV*B z?Nmj{;hBFBNm?n=;^vgRUg>B&RH<+3Awhi7YvEQ3N_Tn8c3s=t>fk)+QkmN$V*5j&2`tl43 z5x27mF_7Uk5;ac3vlT)D&Yv{T0-piw^-Kc|4FMI2A;1Z_TlrHV5<2^N=Es-4#J6fq z7rl)uWmO#$?>>$DEIOb5@ZQ|iA0F#On7!9;VT0^+Z7&?9VK@rbYS!J0NbrVYLsLmd z=lAte$vfOD_GQ6vL5Nw*E1@Kz7sMM+f3x2kyg~{jKn5k5>y`sap;pR2=Lmz%pjY?X zsb!V-c7w>+2g{B}O_XLuaPH-^M6)Tt4BvDp?4QJbdRQ0I_Q=yEb6=@_Qd&2gda>

- - - <%= @inner_content %> -
diff --git a/lib/supavisor_web/templates/layout/live.html.heex b/lib/supavisor_web/templates/layout/live.html.heex deleted file mode 100644 index a29d6044..00000000 --- a/lib/supavisor_web/templates/layout/live.html.heex +++ /dev/null @@ -1,11 +0,0 @@ -
- - - - - <%= @inner_content %> -
diff --git a/lib/supavisor_web/templates/layout/root.html.heex b/lib/supavisor_web/templates/layout/root.html.heex deleted file mode 100644 index cd476343..00000000 --- a/lib/supavisor_web/templates/layout/root.html.heex +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - Supavisor - - - <%= @inner_content %> - - diff --git a/lib/supavisor_web/views/error_helpers.ex b/lib/supavisor_web/views/error_helpers.ex index f6109bea..52951c76 100644 --- a/lib/supavisor_web/views/error_helpers.ex +++ b/lib/supavisor_web/views/error_helpers.ex @@ -3,20 +3,6 @@ defmodule SupavisorWeb.ErrorHelpers do Conveniences for translating and building error messages. """ - use Phoenix.HTML - - @doc """ - Generates tag for inlined form input errors. - """ - def error_tag(form, field) do - Enum.map(Keyword.get_values(form.errors, field), fn error -> - content_tag(:span, translate_error(error), - class: "invalid-feedback", - phx_feedback_for: input_name(form, field) - ) - end) - end - @doc """ Translates an error message using gettext. """ diff --git a/lib/supavisor_web/views/layout_view.ex b/lib/supavisor_web/views/layout_view.ex deleted file mode 100644 index 55eb75d3..00000000 --- a/lib/supavisor_web/views/layout_view.ex +++ /dev/null @@ -1,7 +0,0 @@ -defmodule SupavisorWeb.LayoutView do - use SupavisorWeb, :view - - # Phoenix LiveDashboard is available only in development by default, - # so we instruct Elixir to not warn if the dashboard route is missing. - @compile {:no_warn_undefined, {Routes, :live_dashboard_path, 2}} -end diff --git a/mix.exs b/mix.exs index f7ce8382..2d0e9e3b 100644 --- a/mix.exs +++ b/mix.exs @@ -42,7 +42,6 @@ defmodule Supavisor.MixProject do {:phoenix_ecto, "~> 4.4"}, {:ecto_sql, "~> 3.10"}, {:postgrex, ">= 0.0.0"}, - {:phoenix_html, "~> 3.0"}, {:phoenix_view, "~> 2.0.2"}, {:phoenix_live_reload, "~> 1.2", only: :dev}, {:phoenix_live_view, "~> 0.20.0"}, diff --git a/test/supavisor_web/views/layout_view_test.exs b/test/supavisor_web/views/layout_view_test.exs deleted file mode 100644 index 5584a842..00000000 --- a/test/supavisor_web/views/layout_view_test.exs +++ /dev/null @@ -1,8 +0,0 @@ -defmodule SupavisorWeb.LayoutViewTest do - use SupavisorWeb.ConnCase, async: true - - # When testing helpers, you may want to import Phoenix.HTML and - # use functions such as safe_to_string() to convert the helper - # result into an HTML string. - # import Phoenix.HTML -end From 3424615aeabab7e6020edaa57d7c8bd16fae5b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 9 Jul 2024 10:12:33 +0200 Subject: [PATCH 19/97] fix: response type definition (#393) --- lib/supavisor.ex | 2 +- lib/supavisor/client_handler.ex | 4 ++-- lib/supavisor_web/open_api_schemas.ex | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 0745b658..c13307c9 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -75,7 +75,7 @@ defmodule Supavisor do end @spec subscribe_local(pid, id) :: {:ok, subscribe_opts} | {:error, any()} - def(subscribe_local(pid, id)) do + def subscribe_local(pid, id) do with {:ok, workers} <- get_local_workers(id), {:ok, ps, idle_timeout} <- Manager.subscribe(workers.manager, pid) do {:ok, %{workers: workers, ps: ps, idle_timeout: idle_timeout}} diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index a4afbc90..31e5a6f9 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -945,8 +945,8 @@ defmodule Supavisor.ClientHandler do @spec handle_prepared_statements({pid, pid}, binary, map) :: :ok | nil defp handle_prepared_statements({_, pid}, bin, %{mode: :transaction} = data) do with {:ok, payload} <- Client.get_payload(bin), - {:ok, statamets} <- Supavisor.PgParser.statements(payload), - true <- Enum.member?([["PrepareStmt"], ["DeallocateStmt"]], statamets) do + {:ok, statements} <- Supavisor.PgParser.statements(payload), + true <- statements in [["PrepareStmt"], ["DeallocateStmt"]] do Logger.info("ClientHandler: Handle prepared statement #{inspect(payload)}") GenServer.call(data.pool, :get_all_workers) diff --git a/lib/supavisor_web/open_api_schemas.ex b/lib/supavisor_web/open_api_schemas.ex index 3899c82e..37a44b1e 100644 --- a/lib/supavisor_web/open_api_schemas.ex +++ b/lib/supavisor_web/open_api_schemas.ex @@ -203,7 +203,7 @@ defmodule SupavisorWeb.OpenApiSchemas do require OpenApiSpex OpenApiSpex.schema(%{}) - def response(), do: {"", "text/plain", __MODULE__} + def response(), do: {"", "application/json", __MODULE__} end defmodule NotFound do @@ -211,6 +211,6 @@ defmodule SupavisorWeb.OpenApiSchemas do require OpenApiSpex OpenApiSpex.schema(%{}) - def response(), do: {"Not found", "text/plain", __MODULE__} + def response(), do: {"Not found", "application/json", __MODULE__} end end From 8302e792ef36a1a09a4bce5b35db8d1b9536c672 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 10 Jul 2024 10:55:13 +0200 Subject: [PATCH 20/97] chore: dump rustler to 0.34.0 and remove burrito (#395) --- Makefile | 6 ------ mix.exs | 14 +------------- mix.lock | 9 ++++----- 3 files changed, 5 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 8dbbf35d..337fcfe8 100644 --- a/Makefile +++ b/Makefile @@ -43,12 +43,6 @@ dev.node3: ERL_AFLAGS="-kernel shell_history enabled" \ iex --name node3@127.0.0.1 --cookie cookie -S mix phx.server -dev_bin: - MIX_ENV=dev mix release supavisor_bin && ls -l burrito_out - -bin: - MIX_ENV=prod mix release supavisor_bin && ls -l burrito_out - db_migrate: mix ecto.migrate --prefix _supavisor --log-migrator-sql diff --git a/mix.exs b/mix.exs index 2d0e9e3b..6906eaa7 100644 --- a/mix.exs +++ b/mix.exs @@ -60,7 +60,6 @@ defmodule Supavisor.MixProject do # TODO: point it to Supabase fork of prom_ex when available {:prom_ex, github: "hauleth/prom_ex", branch: "ft/add-peep-storage"}, {:open_api_spex, "~> 3.16"}, - {:burrito, github: "burrito-elixir/burrito"}, {:libcluster, "~> 3.3.1"}, {:logflare_logger_backend, github: "Logflare/logflare_logger_backend", tag: "v0.11.4"}, {:distillery, "~> 2.1"}, @@ -74,7 +73,7 @@ defmodule Supavisor.MixProject do {:poolboy, git: "https://github.com/abc3/poolboy.git", tag: "v0.0.2"}, {:syn, "~> 3.3"}, {:pgo, "~> 0.13"}, - {:rustler, "~> 0.33.0"}, + {:rustler, "~> 0.34.0"}, {:ranch, "~> 2.0", override: true} ] end @@ -85,17 +84,6 @@ defmodule Supavisor.MixProject do steps: [:assemble, &upgrade/1, :tar], include_erts: System.get_env("INCLUDE_ERTS", "true") == "true", cookie: System.get_env("RELEASE_COOKIE", Base.url_encode64(:crypto.strong_rand_bytes(30))) - ], - supavisor_bin: [ - steps: [:assemble, &Burrito.wrap/1], - burrito: [ - targets: [ - macos_aarch64: [os: :darwin, cpu: :aarch64], - macos_x86_64: [os: :darwin, cpu: :x86_64], - linux_x86_64: [os: :linux, cpu: :x86_64], - linux_aarch64: [os: :linux, cpu: :aarch64] - ] - ] ] ] end diff --git a/mix.lock b/mix.lock index 5a43cde9..db976b77 100644 --- a/mix.lock +++ b/mix.lock @@ -4,7 +4,6 @@ "benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"}, "bertex": {:hex, :bertex, "1.3.0", "0ad0df9159b5110d9d2b6654f72fbf42a54884ef43b6b651e6224c0af30ba3cb", [:mix], [], "hexpm", "0a5d5e478bb5764b7b7bae37cae1ca491200e58b089df121a2fe1c223d8ee57a"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, - "burrito": {:git, "https://github.com/burrito-elixir/burrito.git", "efd6a329f26e4039e7d46d00e571dc1df5b69262", []}, "cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"}, "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, "cloak": {:hex, :cloak, "1.1.4", "aba387b22ea4d80d92d38ab1890cc528b06e0e7ef2a4581d71c3fdad59e997e7", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "92b20527b9aba3d939fab0dd32ce592ff86361547cfdc87d74edce6f980eb3d7"}, @@ -50,7 +49,7 @@ "pgo": {:hex, :pgo, "0.14.0", "f53711d103d7565db6fc6061fcf4ff1007ab39892439be1bb02d9f686d7e6663", [:rebar3], [{:backoff, "~> 1.1.6", [hex: :backoff, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:pg_types, "~> 0.4.0", [hex: :pg_types, repo: "hexpm", optional: false]}], "hexpm", "71016c22599936e042dc0012ee4589d24c71427d266292f775ebf201d97df9c9"}, "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.2", "3b83b24ab5a2eb071a20372f740d7118767c272db386831b2e77638c4dcc606d", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "3f94d025f59de86be00f5f8c5dd7b5965a3298458d21ab1c328488be3b5fcd59"}, - "phoenix_html": {:hex, :phoenix_html, "3.3.4", "42a09fc443bbc1da37e372a5c8e6755d046f22b9b11343bf885067357da21cb3", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0249d3abec3714aff3415e7ee3d9786cb325be3151e6c4b3021502c585bf53fb"}, + "phoenix_html": {:hex, :phoenix_html, "4.1.1", "4c064fd3873d12ebb1388425a8f2a19348cef56e7289e1998e2d2fa758aa982e", [:mix], [], "hexpm", "f2f2df5a72bc9a2f510b21497fd7d2b86d932ec0598f0210fed4114adc546c6f"}, "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.4", "4508e481f791ce62ec6a096e13b061387158cbeefacca68c6c1928e1305e23ed", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "2984aae96994fbc5c61795a73b8fb58153b41ff934019cfb522343d2d3817d59"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.5.3", "f2161c207fda0e4fb55165f650f7f8db23f02b29e3bff00ff7ef161d6ac1f09d", [:mix], [{:file_system, "~> 0.3 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "b4ec9cd73cb01ff1bd1cac92e045d13e7030330b74164297d1aee3907b54803c"}, "phoenix_live_view": {:hex, :phoenix_live_view, "0.20.17", "f396bbdaf4ba227b82251eb75ac0afa6b3da5e509bc0d030206374237dfc9450", [:mix], [{:floki, "~> 0.36", [hex: :floki, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a61d741ffb78c85fdbca0de084da6a48f8ceb5261a79165b5a0b59e5f65ce98b"}, @@ -65,8 +64,8 @@ "prom_ex": {:git, "https://github.com/hauleth/prom_ex.git", "c9b0793e7483d09a96252992debf30b7bb6b1216", [branch: "ft/add-peep-storage"]}, "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, "recon": {:hex, :recon, "2.5.5", "c108a4c406fa301a529151a3bb53158cadc4064ec0c5f99b03ddb8c0e4281bdf", [:mix, :rebar3], [], "hexpm", "632a6f447df7ccc1a4a10bdcfce71514412b16660fe59deca0fcf0aa3c054404"}, - "req": {:hex, :req, "0.4.0", "1c759054dd64ef1b1a0e475c2d2543250d18f08395d3174c371b7746984579ce", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "f53eadc32ebefd3e5d50390356ec3a59ed2b8513f7da8c6c3f2e14040e9fe989"}, - "rustler": {:hex, :rustler, "0.33.0", "4a5b0a7a7b0b51549bea49947beff6fae9bc5d5326104dcd4531261e876b5619", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "7c4752728fee59a815ffd20c3429c55b644041f25129b29cdeb5c470b80ec5fd"}, + "req": {:hex, :req, "0.5.2", "70b4976e5fbefe84e5a57fd3eea49d4e9aa0ac015301275490eafeaec380f97f", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0c63539ab4c2d6ced6114d2684276cef18ac185ee00674ee9af4b1febba1f986"}, + "rustler": {:hex, :rustler, "0.34.0", "e9a73ee419fc296a10e49b415a2eb87a88c9217aa0275ec9f383d37eed290c1c", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "1d0c7449482b459513003230c0e2422b0252245776fe6fd6e41cb2b11bd8e628"}, "sleeplocks": {:hex, :sleeplocks, "1.1.3", "96a86460cc33b435c7310dbd27ec82ca2c1f24ae38e34f8edde97f756503441a", [:rebar3], [], "hexpm", "d3b3958552e6eb16f463921e70ae7c767519ef8f5be46d7696cc1ed649421321"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, @@ -74,7 +73,7 @@ "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"}, "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, - "tesla": {:hex, :tesla, "1.11.1", "902ec0cd9fb06ba534be765f0eb78acd9d0ef70118230dc3a73fdc9afc91d036", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "c02d7dd149633c55c40adfaad6c3ce2615cfc89258b67a7f428c14bb835c398c"}, + "tesla": {:hex, :tesla, "1.11.2", "24707ac48b52f72f88fc05d242b1c59a85d1ee6f16f19c312d7d3419665c9cd5", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "c549cd03aec6a7196a641689dd378b799e635eb393f689b4bd756f750c7a4014"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, "unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"}, From e0663a5a21ac0170f91d0609f45d7fd385be1d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Wed, 10 Jul 2024 13:21:07 +0200 Subject: [PATCH 21/97] chore: small style cleanups (#396) --- lib/supavisor.ex | 3 --- lib/supavisor/client_handler.ex | 10 +++------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/supavisor.ex b/lib/supavisor.ex index c13307c9..169a0b1f 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -79,9 +79,6 @@ defmodule Supavisor do with {:ok, workers} <- get_local_workers(id), {:ok, ps, idle_timeout} <- Manager.subscribe(workers.manager, pid) do {:ok, %{workers: workers, ps: ps, idle_timeout: idle_timeout}} - else - error -> - error end end diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 31e5a6f9..6084f79d 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -630,13 +630,9 @@ defmodule Supavisor.ClientHandler do def terminate(reason, _state, %{db_pid: {_, pid}}) do db_info = - case Db.get_state_and_mode(pid) do - {:ok, {state, mode} = resp} -> - if state == :busy or mode == :session, do: Db.stop(pid) - resp - - error -> - error + with {:ok, {state, mode} = resp} <- Db.get_state_and_mode(pid) do + if state == :busy or mode == :session, do: Db.stop(pid) + resp end Logger.warning( From 95dc53c84f6376d7b95f021e83980645fc04aeee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Wed, 10 Jul 2024 15:52:03 +0200 Subject: [PATCH 22/97] chore: fix typing errors (#397) I have used [`typos`][typos] tool to find all typos in source code over the codebase. [typos]: https://github.com/crate-ci/typos --- docker-compose.db.yml | 4 ++-- docker-compose.yml | 4 ++-- docs/configuration/tenants.md | 2 +- docs/connecting/authentication.md | 2 +- docs/development/profiling.md | 2 +- docs/faq.md | 2 +- docs/migrating/pgbouncer.md | 2 +- docs/orms/prisma.md | 4 ++-- lib/supavisor/client_handler.ex | 4 +++- lib/supavisor/helpers.ex | 4 ++-- lib/supavisor_web/controllers/tenant_controller.ex | 2 +- test/supavisor/db_handler_test.exs | 4 ++-- typos.toml | 5 +++++ 13 files changed, 24 insertions(+), 17 deletions(-) create mode 100644 typos.toml diff --git a/docker-compose.db.yml b/docker-compose.db.yml index 5d7e9150..16bdf76a 100644 --- a/docker-compose.db.yml +++ b/docker-compose.db.yml @@ -8,11 +8,11 @@ services: - "6432:5432" volumes: - ./dev/postgres:/docker-entrypoint-initdb.d/ - # Uncomment to set MD5 authentication method on unitialized databases + # Uncomment to set MD5 authentication method on uninitialized databases # - ./dev/postgres/md5/etc/postgresql/pg_hba.conf:/etc/postgresql/pg_hba.conf command: postgres -c config_file=/etc/postgresql/postgresql.conf environment: POSTGRES_HOST: /var/run/postgresql POSTGRES_PASSWORD: postgres - # Uncomment to set MD5 authentication method on unitialized databases + # Uncomment to set MD5 authentication method on uninitialized databases # POSTGRES_INITDB_ARGS: --auth-host=md5 diff --git a/docker-compose.yml b/docker-compose.yml index cd12697a..8c1eaf98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,13 +8,13 @@ services: - "6432:5432" volumes: - ./dev/postgres:/docker-entrypoint-initdb.d/ - # Uncomment to set MD5 authentication method on unitialized databases + # Uncomment to set MD5 authentication method on uninitialized databases # - ./dev/postgres/md5/etc/postgresql/pg_hba.conf:/etc/postgresql/pg_hba.conf command: postgres -c config_file=/etc/postgresql/postgresql.conf environment: POSTGRES_HOST: /var/run/postgresql POSTGRES_PASSWORD: postgres - # Uncomment to set MD5 authentication method on unitialized databases + # Uncomment to set MD5 authentication method on uninitialized databases # POSTGRES_INITDB_ARGS: --auth-host=md5 supavisor: build: . diff --git a/docs/configuration/tenants.md b/docs/configuration/tenants.md index 595650cc..4e1560b2 100644 --- a/docs/configuration/tenants.md +++ b/docs/configuration/tenants.md @@ -33,7 +33,7 @@ server `require_user` - require client connection credentials to match `user` credentials in the metadata database -`auth_query` - the query to use when matching credential agains a client +`auth_query` - the query to use when matching credential against a client connection `default_pool_size` - the default size of the database pool diff --git a/docs/connecting/authentication.md b/docs/connecting/authentication.md index 24c1cb0f..f199e48f 100644 --- a/docs/connecting/authentication.md +++ b/docs/connecting/authentication.md @@ -1,7 +1,7 @@ When a client connection is established Supavisor needs to verify the credentials of the connection. -Credential verificiation is done either via `user` records or an `auth_query`. +Credential verification is done either via `user` records or an `auth_query`. ## Tenant User Record diff --git a/docs/development/profiling.md b/docs/development/profiling.md index 2cb63293..35e926e2 100644 --- a/docs/development/profiling.md +++ b/docs/development/profiling.md @@ -4,7 +4,7 @@ Example profiling session looks like: - Start application within IEx session (for example by using `make dev`) - Within given session you can specify which function you want to trace, by - calling `:eflambe.capture({mod, func, arity}, no_of_caputres)`, however it is + calling `:eflambe.capture({mod, func, arity}, no_of_captures)`, however it is useful to have some separate directory to store all traces, for that one can use quick snippet diff --git a/docs/faq.md b/docs/faq.md index 720b39e4..31a6096b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -36,5 +36,5 @@ the tenant. Also running N pools on N nodes for N clients will not scale horizontally as well because all nodes will be doing all the same work of issuing database connections to clients. While not a lot of overhead, at some point this won't -scale and we'd have to run multiple independant clusters and route tenants to +scale and we'd have to run multiple independent clusters and route tenants to clusters to scale horizontally. diff --git a/docs/migrating/pgbouncer.md b/docs/migrating/pgbouncer.md index 02fe0af7..66dbae54 100644 --- a/docs/migrating/pgbouncer.md +++ b/docs/migrating/pgbouncer.md @@ -29,7 +29,7 @@ select count(*) from pg_stat_activity; ## Change Postgres `max_connections` Based on the responses above configure the `default_pool_size` accordingly or -increase your `max_connections` limit on Postgres to accomadate two connection +increase your `max_connections` limit on Postgres to accommodate two connection poolers. e.g if you're using 30 connections out of 100 and you set your diff --git a/docs/orms/prisma.md b/docs/orms/prisma.md index c15e887e..9396f535 100644 --- a/docs/orms/prisma.md +++ b/docs/orms/prisma.md @@ -1,6 +1,6 @@ Connecting to a Postgres database with Prisma is easy. -## PgBouncer Compatability +## PgBouncer Compatibility Supavisor pool modes behave the same way as PgBouncer. You should be able to connect to Supavisor with the exact same connection string as you use for @@ -13,7 +13,7 @@ Prisma will use named prepared statements to query Postgres by default. To turn off named prepared statements use `pgbouncer=true` in your connection string with Prisma. -The `pgbouncer=true` connection string parameter is compatable with Supavisor. +The `pgbouncer=true` connection string parameter is compatible with Supavisor. ## Prisma Connection Management diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 6084f79d..aa8b270d 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -576,7 +576,9 @@ defmodule Supavisor.ClientHandler do :keep_state_and_data :read_sql_error -> - Logger.error("ClientHandler: read only sql transaction, reruning the query to write pool") + Logger.error( + "ClientHandler: read only sql transaction, rerunning the query to write pool" + ) # release the read pool _ = handle_db_pid(data.mode, data.pool, data.db_pid) diff --git a/lib/supavisor/helpers.ex b/lib/supavisor/helpers.ex index 895a2c88..d64ec52a 100644 --- a/lib/supavisor/helpers.ex +++ b/lib/supavisor/helpers.ex @@ -101,9 +101,9 @@ defmodule Supavisor.Helpers do {:error, "There is no user '#{user}' in the database. Please create it or change the user in the config"} - %{columns: colums} -> + %{columns: columns} -> {:error, - "Authentification query returned wrong format. Should be two columns: user and secret, but got: #{inspect(colums)}"} + "Authentication query returned wrong format. Should be two columns: user and secret, but got: #{inspect(columns)}"} {:error, reason} -> {:error, reason} diff --git a/lib/supavisor_web/controllers/tenant_controller.ex b/lib/supavisor_web/controllers/tenant_controller.ex index 6c7c82e7..3a5d867f 100644 --- a/lib/supavisor_web/controllers/tenant_controller.ex +++ b/lib/supavisor_web/controllers/tenant_controller.ex @@ -89,7 +89,7 @@ defmodule SupavisorWeb.TenantController do } ) - # conver cert to pem format + # convert cert to pem format def update(conn, %{ "external_id" => id, "tenant" => %{"upstream_tls_ca" => "-----BEGIN" <> _ = upstream_tls_ca} = tenant_params diff --git a/test/supavisor/db_handler_test.exs b/test/supavisor/db_handler_test.exs index 417b72f2..efccc86a 100644 --- a/test/supavisor/db_handler_test.exs +++ b/test/supavisor/db_handler_test.exs @@ -33,7 +33,7 @@ defmodule Supavisor.DbHandlerTest do end describe "handle_event/4" do - test "db is avaible" do + test "db is available" do :meck.new(:gen_tcp, [:unstick, :passthrough]) :meck.new(:inet, [:unstick, :passthrough]) :meck.expect(:gen_tcp, :connect, fn _host, _port, _sock_opts -> {:ok, :sock} end) @@ -80,7 +80,7 @@ defmodule Supavisor.DbHandlerTest do :meck.unload(:gen_tcp) end - test "db is not avaible" do + test "db is not available" do :meck.new(:gen_tcp, [:unstick, :passthrough]) :meck.expect(:gen_tcp, :connect, fn _host, _port, _sock_opts -> {:error, "some error"} end) diff --git a/typos.toml b/typos.toml new file mode 100644 index 00000000..6b76359c --- /dev/null +++ b/typos.toml @@ -0,0 +1,5 @@ +[default] +extend-ignore-re = [ + "\\bey[A-Za-z0-9_-]{20,}\\.ey[A-Za-z0-9_-]{20,}\\.[A-Za-z0-9_-]{20,}\\b", + "\\bfly-local-[a-z0-9]+\\b" +] From 357ca080c240d231c6e3aa03a36ef4d5c310c07f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Fri, 12 Jul 2024 15:37:00 +0200 Subject: [PATCH 23/97] chore: move to new deps heads after rebases (#399) --- mix.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index db976b77..40e65256 100644 --- a/mix.lock +++ b/mix.lock @@ -44,7 +44,7 @@ "open_api_spex": {:hex, :open_api_spex, "3.19.1", "65ccb5d06e3d664d1eec7c5ea2af2289bd2f37897094a74d7219fb03fc2b5994", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "392895827ce2984a3459c91a484e70708132d8c2c6c5363972b4b91d6bbac3dd"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.3.0", "03e2177f28dd8d11aaa88e8522c81c2f6a788170fe52f7a65262340961e663f9", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "b9e5ff775fd064fa098dba3c398490b77649a352b40b0b730a6b7dc0bdd68858"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, - "peep": {:git, "https://github.com/hauleth/peep.git", "d2e30ba21e8937bd00c10c19488cf1d111a3a39f", [branch: "ft/custom-prometheus-types"]}, + "peep": {:git, "https://github.com/hauleth/peep.git", "7f640c06e3ee4ce4f4e5dd9f262cb3377dfcb1ce", [branch: "ft/custom-prometheus-types"]}, "pg_types": {:hex, :pg_types, "0.4.0", "3ce365c92903c5bb59c0d56382d842c8c610c1b6f165e20c4b652c96fa7e9c14", [:rebar3], [], "hexpm", "b02efa785caececf9702c681c80a9ca12a39f9161a846ce17b01fb20aeeed7eb"}, "pgo": {:hex, :pgo, "0.14.0", "f53711d103d7565db6fc6061fcf4ff1007ab39892439be1bb02d9f686d7e6663", [:rebar3], [{:backoff, "~> 1.1.6", [hex: :backoff, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:pg_types, "~> 0.4.0", [hex: :pg_types, repo: "hexpm", optional: false]}], "hexpm", "71016c22599936e042dc0012ee4589d24c71427d266292f775ebf201d97df9c9"}, "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"}, @@ -61,7 +61,7 @@ "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "poolboy": {:git, "https://github.com/abc3/poolboy.git", "999ec7f5c7282d515020bb058b4832029d6d07bc", [tag: "v0.0.2"]}, "postgrex": {:hex, :postgrex, "0.18.0", "f34664101eaca11ff24481ed4c378492fed2ff416cd9b06c399e90f321867d7e", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a042989ba1bc1cca7383ebb9e461398e3f89f868c92ce6671feb7ef132a252d1"}, - "prom_ex": {:git, "https://github.com/hauleth/prom_ex.git", "c9b0793e7483d09a96252992debf30b7bb6b1216", [branch: "ft/add-peep-storage"]}, + "prom_ex": {:git, "https://github.com/hauleth/prom_ex.git", "34b714f8619c7637c728e3f385eb1cf1e75dc2fb", [branch: "ft/add-peep-storage"]}, "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, "recon": {:hex, :recon, "2.5.5", "c108a4c406fa301a529151a3bb53158cadc4064ec0c5f99b03ddb8c0e4281bdf", [:mix, :rebar3], [], "hexpm", "632a6f447df7ccc1a4a10bdcfce71514412b16660fe59deca0fcf0aa3c054404"}, "req": {:hex, :req, "0.5.2", "70b4976e5fbefe84e5a57fd3eea49d4e9aa0ac015301275490eafeaec380f97f", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0c63539ab4c2d6ced6114d2684276cef18ac185ee00674ee9af4b1febba1f986"}, From b980c11c4a5772a3cf385f4a13570bfe7f667c56 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 17 Jul 2024 12:42:40 +0200 Subject: [PATCH 24/97] feat: implement proxy handler (#401) --- config/config.exs | 12 +- config/dev.exs | 6 +- lib/supavisor/application.ex | 11 +- lib/supavisor/handler_helpers.ex | 20 +- lib/supavisor/handlers/proxy/client.ex | 662 ++++++++++++++++++++++++ lib/supavisor/handlers/proxy/db.ex | 260 ++++++++++ lib/supavisor/handlers/proxy/handler.ex | 154 ++++++ lib/supavisor/helpers.ex | 8 + lib/supavisor/monitoring/prom_ex.ex | 3 +- 9 files changed, 1116 insertions(+), 20 deletions(-) create mode 100644 lib/supavisor/handlers/proxy/client.ex create mode 100644 lib/supavisor/handlers/proxy/db.ex create mode 100644 lib/supavisor/handlers/proxy/handler.ex diff --git a/config/config.exs b/config/config.exs index 7255b2ec..c0bc7c94 100644 --- a/config/config.exs +++ b/config/config.exs @@ -23,7 +23,17 @@ config :supavisor, SupavisorWeb.Endpoint, # Configures Elixir's Logger config :logger, :console, format: "$time $metadata[$level] $message\n", - metadata: [:request_id, :project, :user, :region, :instance_id, :mode, :type] + metadata: [ + :request_id, + :project, + :user, + :region, + :instance_id, + :mode, + :type, + :app_name, + :peer_ip + ] # Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason diff --git a/config/dev.exs b/config/dev.exs index 2e30a3bc..af12753d 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -61,9 +61,9 @@ config :supavisor, SupavisorWeb.Endpoint, # Configures Elixir's Logger config :logger, :console, format: "$time [$level] $message $metadata\n", - # level: :debug, - level: :notice, - metadata: [:error_code, :file, :line, :pid, :project, :user, :mode, :type] + level: :debug, + # level: :notice, + metadata: [:error_code, :file, :line, :pid, :project, :user, :mode, :type, :app_name, :peer_ip] # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive. diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index 11ed773c..760ef2ef 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -6,6 +6,8 @@ defmodule Supavisor.Application do use Application require Logger alias Supavisor.Monitoring.PromEx + alias Supavisor.ClientHandler + alias Supavisor.Handlers.Proxy.Handler, as: ProxyHandler @impl true def start(_type, _args) do @@ -29,11 +31,12 @@ defmodule Supavisor.Application do proxy_ports = [ {:pg_proxy_transaction, Application.get_env(:supavisor, :proxy_port_transaction), - :transaction}, - {:pg_proxy_session, Application.get_env(:supavisor, :proxy_port_session), :session} + :transaction, ClientHandler}, + {:pg_proxy_session, Application.get_env(:supavisor, :proxy_port_session), :session, + ProxyHandler} ] - for {key, port, mode} <- proxy_ports do + for {key, port, mode, handler} <- proxy_ports do case :ranch.start_listener( key, :ranch_tcp, @@ -42,7 +45,7 @@ defmodule Supavisor.Application do num_acceptors: String.to_integer(System.get_env("NUM_ACCEPTORS") || "100"), socket_opts: [inet_backend: :socket, port: port, keepalive: true] }, - Supavisor.ClientHandler, + handler, %{mode: mode} ) do {:ok, _pid} -> diff --git a/lib/supavisor/handler_helpers.ex b/lib/supavisor/handler_helpers.ex index 58c914a8..a7b58496 100644 --- a/lib/supavisor/handler_helpers.ex +++ b/lib/supavisor/handler_helpers.ex @@ -10,8 +10,9 @@ defmodule Supavisor.HandlerHelpers do mod.send(sock, data) end - @spec sock_close(nil | S.sock()) :: :ok | {:error, term()} + @spec sock_close(S.sock() | nil | {any(), nil}) :: :ok | {:error, term()} def sock_close(nil), do: :ok + def sock_close({_, nil}), do: :ok def sock_close({mod, sock}) do mod.close(sock) @@ -23,14 +24,11 @@ defmodule Supavisor.HandlerHelpers do mod.setopts(sock, opts) end - @spec activate(S.sock()) :: :ok | {:error, term} - def activate({:gen_tcp, sock}) do - :inet.setopts(sock, active: true) - end + @spec active_once(S.sock()) :: :ok | {:error, term} + def active_once(sock), do: setopts(sock, active: :once) - def activate({:ssl, sock}) do - :ssl.setopts(sock, active: true) - end + @spec activate(S.sock()) :: :ok | {:error, term} + def activate(sock), do: setopts(sock, active: true) @spec try_ssl_handshake(S.tcp_sock(), boolean) :: {:ok, S.sock()} | {:error, term()} @@ -106,12 +104,12 @@ defmodule Supavisor.HandlerHelpers do end end - @spec send_cancel_query(non_neg_integer, non_neg_integer) :: :ok | {:errr, term} - def send_cancel_query(pid, key) do + @spec send_cancel_query(non_neg_integer, non_neg_integer, term) :: :ok | {:errr, term} + def send_cancel_query(pid, key, msg \\ :cancel_query) do PubSub.broadcast( Supavisor.PubSub, "cancel_req:#{pid}_#{key}", - :cancel_query + msg ) end diff --git a/lib/supavisor/handlers/proxy/client.ex b/lib/supavisor/handlers/proxy/client.ex new file mode 100644 index 00000000..7a157fdf --- /dev/null +++ b/lib/supavisor/handlers/proxy/client.ex @@ -0,0 +1,662 @@ +defmodule Supavisor.Handlers.Proxy.Client do + @moduledoc false + + require Logger + + alias Supavisor, as: S + alias Supavisor.ProxyDb, as: Db + alias Supavisor.Helpers, as: H + alias Supavisor.HandlerHelpers, as: HH + + alias Supavisor.{ + Tenants, + ProxyHandlerDb, + Monitoring.Telem, + Protocol.Client, + Protocol.Server + } + + alias Supavisor.Handlers.Proxy.Db, as: ProxyDb + + @cancel_query_msg <<16::32, 1234::16, 5678::16>> + @sock_closed [:tcp_closed, :ssl_closed] + @proto [:tcp, :ssl] + + def handle_event(:info, {_proto, _, <<"GET", _::binary>>}, :exchange, data) do + Logger.debug("ProxyClient: Client is trying to request HTTP") + HH.sock_send(data.sock, "HTTP/1.1 204 OK\r\nx-app-version: #{data.version}\r\n\r\n") + {:stop, {:shutdown, :http_request}} + end + + # cancel request + def handle_event(:info, {_, _, <<@cancel_query_msg, pid::32, key::32>>}, _, _) do + Logger.debug("ProxyClient: Got cancel query for #{inspect({pid, key})}") + :ok = HH.send_cancel_query(pid, key, {:client, :cancel_query}) + {:stop, {:shutdown, :cancel_query}} + end + + def handle_event(:info, {:client, :cancel_query}, _, %{ + auth: auth, + backend_key_data: b + }) do + :ok = HH.cancel_query(~c"#{auth.host}", auth.port, auth.ip_ver, b.pid, b.key) + :keep_state_and_data + end + + # # ssl request from client + def handle_event(:info, {:tcp, _, <<_::64>>}, :exchange, %{sock: sock} = data) do + Logger.debug("ProxyClient: Client is trying to connect with SSL") + + downstream_cert = H.downstream_cert() + downstream_key = H.downstream_key() + + # SSL negotiation, S/N/Error + if !!downstream_cert and !!downstream_key do + :ok = HH.setopts(sock, active: false) + :ok = HH.sock_send(sock, "S") + + opts = [ + certfile: downstream_cert, + keyfile: downstream_key + ] + + case :ssl.handshake(elem(sock, 1), opts) do + {:ok, ssl_sock} -> + socket = {:ssl, ssl_sock} + :ok = HH.setopts(socket, active: true) + {:keep_state, %{data | sock: socket, ssl: true}} + + error -> + Logger.error("ProxyClient: SSL handshake error: #{inspect(error)}") + Telem.client_join(:fail, data.id) + {:stop, {:shutdown, :ssl_handshake_error}} + end + else + Logger.error("ProxyClient: User requested SSL connection but no downstream cert/key found") + + :ok = HH.sock_send(data.sock, "N") + :keep_state_and_data + end + end + + def handle_event(:info, {_, _, bin}, :exchange, data) do + case Server.decode_startup_packet(bin) do + {:ok, hello} -> + Logger.debug("ProxyClient: Client startup message: #{inspect(hello)}") + {type, {user, tenant_or_alias, db_name}} = HH.parse_user_info(hello.payload) + + # Validate user and db_name according to PostgreSQL rules. + # The rules are: 1-63 characters, alphanumeric, underscore and $ + # TODO: spaces are allowed in db_name, but we don't support it yet + rule = ~r/^[a-z_][a-z0-9_$]*$/ + + if user =~ rule and db_name =~ rule do + log_level = maybe_change_log(hello) + event = {:hello, {type, {user, tenant_or_alias, db_name}}} + app_name = app_name(hello.payload["application_name"]) + + {:keep_state, %{data | log_level: log_level, app_name: app_name}, + {:next_event, :internal, {:client, event}}} + else + reason = "Invalid format for user or db_name" + Logger.error("ProxyClient: #{inspect(reason)}") + Telem.client_join(:fail, tenant_or_alias) + HH.send_error(data.sock, "XX000", "Authentication error, reason: #{inspect(reason)}") + {:stop, {:shutdown, :invalid_format}} + end + + {:error, error} -> + Logger.error("ProxyClient: Client startup message error: #{inspect(error)}") + Telem.client_join(:fail, data.id) + {:stop, {:shutdown, :startup_packet_error}} + end + end + + def handle_event( + :internal, + {:client, {:hello, {type, {user, tenant_or_alias, db_name}}}}, + :exchange, + %{sock: sock} = data + ) do + sni_hostname = HH.try_get_sni(sock) + + case Tenants.get_user_cache(type, user, tenant_or_alias, sni_hostname) do + {:ok, info} -> + db_name = if(db_name != nil, do: db_name, else: info.tenant.db_database) + + id = + Supavisor.id( + {type, tenant_or_alias}, + user, + data.mode, + info.user.mode_type, + db_name + ) + + mode = S.mode(id) + + Logger.metadata( + project: tenant_or_alias, + user: user, + mode: mode, + type: type, + db_name: db_name, + app_name: data.app_name, + peer_ip: data.peer_ip + ) + + Registry.register(Supavisor.Registry.TenantClients, id, []) + + {:ok, addr} = HH.addr_from_sock(sock) + + cond do + info.tenant.enforce_ssl and !data.ssl -> + Logger.error( + "ProxyClient: Tenant is not allowed to connect without SSL, user #{user}" + ) + + :ok = HH.send_error(sock, "XX000", "SSL connection is required") + Telem.client_join(:fail, id) + {:stop, {:shutdown, :ssl_required}} + + HH.filter_cidrs(info.tenant.allow_list, addr) == [] -> + message = "Address not in tenant allow_list: " <> inspect(addr) + Logger.error("ProxyClient: #{message}") + :ok = HH.send_error(sock, "XX000", message) + + Telem.client_join(:fail, id) + {:stop, {:shutdown, :address_not_allowed}} + + true -> + new_data = update_user_data(data, info, user, id, db_name, mode) + + key = {:secrets, tenant_or_alias, user} + + case auth_secrets(info, user, key, :timer.hours(24)) do + {:ok, auth_secrets} -> + Logger.debug("ProxyClient: Authentication method: #{inspect(auth_secrets)}") + + event = {:handle, auth_secrets, info} + {:keep_state, new_data, {:next_event, :internal, {:client, event}}} + + {:error, reason} -> + Logger.error("ProxyClient: Authentication auth_secrets error: #{inspect(reason)}") + + :ok = + HH.send_error(sock, "XX000", "Authentication error, reason: #{inspect(reason)}") + + Telem.client_join(:fail, id) + {:stop, {:shutdown, :auth_secrets_error}} + end + end + + {:error, reason} -> + Logger.error( + "ProxyClient: User not found: #{inspect(reason)} #{inspect({type, user, tenant_or_alias})}" + ) + + :ok = HH.send_error(sock, "XX000", "Tenant or user not found") + Telem.client_join(:fail, data.id) + {:stop, {:shutdown, :user_not_found}} + end + end + + def handle_event( + :internal, + {:client, {:handle, {method, secrets}, info}}, + _, + %{sock: sock} = data + ) do + Logger.debug("ProxyClient: Handle exchange, auth method: #{inspect(method)}") + + case handle_exchange(sock, {method, secrets}) do + {:error, reason} -> + Logger.error( + "ProxyClient: Exchange error: #{inspect(reason)} when method #{inspect(method)}" + ) + + msg = + if method == :auth_query_md5, + do: Server.error_message("XX000", reason), + else: Server.exchange_message(:final, "e=#{reason}") + + key = {:secrets_check, data.tenant, data.user} + + if method != :password and reason == "Wrong password" and + Cachex.get(Supavisor.Cache, key) == {:ok, nil} do + case auth_secrets(info, data.user, key, 15_000) do + {:ok, {method2, secrets2}} = value -> + if method != method2 or Map.delete(secrets.(), :client_key) != secrets2.() do + Logger.warning("ProxyClient: Update secrets and terminate pool") + + Cachex.update( + Supavisor.Cache, + {:secrets, data.tenant, data.user}, + {:cached, value} + ) + + Supavisor.stop(data.id) + else + Logger.debug("ProxyClient: Cache the same #{inspect(key)}") + end + + other -> + Logger.error("ProxyClient: Auth secrets check error: #{inspect(other)}") + end + else + Logger.debug("ProxyClient: Cache hit for #{inspect(key)}") + end + + HH.sock_send(sock, msg) + Telem.client_join(:fail, data.id) + {:stop, {:shutdown, :exchange_error}} + + {:ok, client_key} -> + secrets = + if client_key do + fn -> + Map.put(secrets.(), :client_key, client_key) + end + else + secrets + end + + Logger.debug("ProxyClient: Exchange success") + :ok = HH.sock_send(sock, Server.authentication_ok()) + Telem.client_join(:ok, data.id) + + auth = Map.merge(data.auth, %{secrets: secrets, method: method}) + + {:keep_state, %{data | auth_secrets: {method, secrets}, auth: auth}, + {:next_event, :internal, {:client, :connect_db}}} + end + end + + def handle_event(:internal, {:client, :connect_db}, _, %{auth: auth} = data) do + Logger.debug("Try to connect to DB") + Telem.handler_action(:db_handler, :db_connection, data.id) + ip_ver = H.ip_version(auth.ip_version, auth.host) + + sock_opts = [ + :binary, + {:packet, :raw}, + {:active, false}, + {:nodelay, true}, + ip_ver + ] + + case :gen_tcp.connect(~c"#{auth.host}", auth.port, sock_opts) do + {:ok, sock} -> + Logger.debug("ProxyClient: auth #{inspect(auth, pretty: true)}") + + case ProxyDb.try_ssl_handshake({:gen_tcp, sock}, auth) do + {:ok, sock} -> + case ProxyDb.send_startup(sock, auth) do + :ok -> + HH.active_once(sock) + auth = Map.put(auth, :ip_ver, ip_ver) + {:next_state, :db_authentication, %{data | db_sock: sock, auth: auth}} + + {:error, reason} -> + Logger.error("ProxyClient: Send startup error #{inspect(reason)}") + {:stop, {:shutdown, :startup_error}} + end + + {:error, error} -> + Logger.error("ProxyClient: Handshake error #{inspect(error)}") + {:stop, {:shutdown, :handshake_error}} + end + + other -> + Logger.error( + "ProxyClient: Connection failed #{inspect(other)} to #{inspect(auth.host)}:#{inspect(auth.port)}" + ) + + {:stop, {:shutdown, :connection_failed}} + end + end + + def handle_event(:internal, {:client, {:greetings, ps}}, _, %{sock: sock} = data) do + {header, <> = payload} = Server.backend_key_data() + msg = [ps, [header, payload], Server.ready_for_query()] + :ok = HH.listen_cancel_query(pid, key) + :ok = HH.sock_send(sock, msg) + HH.active_once(sock) + Telem.client_connection_time(data.connection_start, data.id) + {:next_state, :idle, data, handle_actions(data)} + end + + def handle_event(:timeout, :subscribe, _, _) do + {:keep_state_and_data, {:next_event, :internal, {:client, :connect_db}}} + end + + def handle_event(:timeout, :wait_ps, _, data) do + Logger.error("ProxyClient: Wait parameter status timeout, send default #{inspect(data.ps)}}") + + ps = Server.encode_parameter_status(data.ps) + {:keep_state_and_data, {:next_event, :internal, {:client, {:greetings, ps}}}} + end + + def handle_event(:timeout, :idle_terminate, _, data) do + Logger.warning("ProxyClient: Terminate an idle connection by #{data.idle_timeout} timeout") + {:stop, {:shutdown, :idle_terminate}} + end + + def handle_event(:timeout, :heartbeat_check, _, data) do + Logger.debug("ProxyClient: Send heartbeat to client") + HH.sock_send(data.sock, Server.application_name()) + {:keep_state_and_data, {:timeout, data.heartbeat_interval, :heartbeat_check}} + end + + # forwards the message to the db + def handle_event(:info, {proto, _, bin}, _, data) when proto in @proto do + HH.sock_send(data.db_sock, bin) + HH.active_once(data.sock) + :keep_state_and_data + end + + def handle_event(:info, {:parameter_status, ps}, :exchange, _), + do: {:keep_state_and_data, {:next_event, :internal, {:client, {:greetings, ps}}}} + + # client closed connection + def handle_event(_, {closed, _}, _, data) + when closed in [:tcp_closed, :ssl_closed] do + Logger.debug("ProxyClient: #{closed} socket closed for #{inspect(data.tenant)}") + {:stop, {:shutdown, :socket_closed}} + end + + ## Internal functions + + @spec handle_exchange(S.sock(), {atom(), fun()}) :: {:ok, binary() | nil} | {:error, String.t()} + def handle_exchange({_, socket} = sock, {:auth_query_md5 = method, secrets}) do + salt = :crypto.strong_rand_bytes(4) + :ok = HH.sock_send(sock, Server.md5_request(salt)) + + with {:ok, + %{ + tag: :password_message, + payload: {:md5, client_md5} + }, _} <- receive_next(socket, "Timeout while waiting for the md5 exchange"), + {:ok, key} <- authenticate_exchange(method, client_md5, secrets.().secret, salt) do + {:ok, key} + else + {:error, message} -> {:error, message} + other -> {:error, "Unexpected message #{inspect(other)}"} + end + end + + def handle_exchange({_, socket} = sock, {method, secrets}) do + :ok = HH.sock_send(sock, Server.scram_request()) + + with {:ok, + %{ + tag: :password_message, + payload: {:scram_sha_256, %{"n" => user, "r" => nonce, "c" => channel}} + }, + _} <- + receive_next( + socket, + "Timeout while waiting for the first password message" + ), + {:ok, signatures} = reply_first_exchange(sock, method, secrets, channel, nonce, user), + {:ok, + %{ + tag: :password_message, + payload: {:first_msg_response, %{"p" => p}} + }, + _} <- + receive_next( + socket, + "Timeout while waiting for the second password message" + ), + {:ok, key} <- authenticate_exchange(method, secrets, signatures, p) do + message = "v=#{Base.encode64(signatures.server)}" + :ok = HH.sock_send(sock, Server.exchange_message(:final, message)) + {:ok, key} + else + {:error, message} -> {:error, message} + other -> {:error, "Unexpected message #{inspect(other)}"} + end + end + + def receive_next(socket, timeout_message) do + receive do + {_proto, ^socket, bin} -> Server.decode_pkt(bin) + other -> {:error, "Unexpected message in receive_next/2 #{inspect(other)}"} + after + 15_000 -> {:error, timeout_message} + end + end + + def reply_first_exchange(sock, method, secrets, channel, nonce, user) do + {message, signatures} = exchange_first(method, secrets, nonce, user, channel) + :ok = HH.sock_send(sock, Server.exchange_message(:first, message)) + {:ok, signatures} + end + + def authenticate_exchange(:password, _secrets, signatures, p) do + if p == signatures.client, + do: {:ok, nil}, + else: {:error, "Wrong password"} + end + + def authenticate_exchange(:auth_query, secrets, signatures, p) do + client_key = :crypto.exor(Base.decode64!(p), signatures.client) + + if H.hash(client_key) == secrets.().stored_key, + do: {:ok, client_key}, + else: {:error, "Wrong password"} + end + + def authenticate_exchange(:auth_query_md5, client_hash, server_hash, salt) do + if "md5" <> H.md5([server_hash, salt]) == client_hash, + do: {:ok, nil}, + else: {:error, "Wrong password"} + end + + @spec update_user_data(map(), map(), String.t(), S.id(), String.t(), S.mode()) :: map() + def update_user_data(data, info, user, id, db_name, mode) do + proxy_type = if info.tenant.require_user, do: :password, else: :auth_query + + auth = %{ + application_name: data.app_name, + database: info.tenant.db_database, + host: info.tenant.db_host, + sni_host: info.tenant.sni_hostname, + ip_version: info.tenant.ip_version, + port: info.tenant.db_port, + user: user, + password: info.user.db_password, + require_user: info.tenant.require_user, + upstream_ssl: info.tenant.upstream_ssl, + upstream_tls_ca: info.tenant.upstream_tls_ca, + upstream_verify: info.tenant.upstream_verify + } + + %{ + data + | tenant: info.tenant.external_id, + user: user, + timeout: info.user.pool_checkout_timeout, + ps: info.tenant.default_parameter_status, + proxy_type: proxy_type, + id: id, + heartbeat_interval: info.tenant.client_heartbeat_interval * 1000, + db_name: db_name, + mode: mode, + auth: auth + } + end + + @spec auth_secrets(map, String.t(), term(), non_neg_integer()) :: + {:ok, S.secrets()} | {:error, term()} + ## password secrets + def auth_secrets(%{user: user, tenant: %{require_user: true}}, _, _, _) do + secrets = %{db_user: user.db_user, password: user.db_password, alias: user.db_user_alias} + {:ok, {:password, fn -> secrets end}} + end + + ## auth_query secrets + def auth_secrets(info, db_user, key, ttl) do + fetch = fn _key -> + case get_secrets(info, db_user) do + {:ok, _} = resp -> {:commit, {:cached, resp}, ttl: ttl} + {:error, _} = resp -> {:ignore, resp} + end + end + + case Cachex.fetch(Supavisor.Cache, key, fetch) do + {:ok, {:cached, value}} -> value + {:commit, {:cached, value}, _opts} -> value + {:ignore, resp} -> resp + end + end + + @spec get_secrets(map, String.t()) :: {:ok, {:auth_query, fun()}} | {:error, term()} + def get_secrets(%{user: user, tenant: tenant}, db_user) do + ssl_opts = + if tenant.upstream_ssl and tenant.upstream_verify == "peer" do + [ + {:verify, :verify_peer}, + {:cacerts, [H.upstream_cert(tenant.upstream_tls_ca)]}, + {:server_name_indication, String.to_charlist(tenant.db_host)}, + {:customize_hostname_check, [{:match_fun, fn _, _ -> true end}]} + ] + end + + {:ok, conn} = + Postgrex.start_link( + hostname: tenant.db_host, + port: tenant.db_port, + database: tenant.db_database, + password: user.db_password, + username: user.db_user, + parameters: [application_name: "Supavisor auth_query"], + ssl: tenant.upstream_ssl, + socket_options: [ + H.ip_version(tenant.ip_version, tenant.db_host) + ], + queue_target: 1_000, + queue_interval: 5_000, + ssl_opts: ssl_opts || [] + ) + + # kill the postgrex connection if the current process exits unexpectedly + Process.link(conn) + + Logger.debug( + "ProxyClient: Connected to db #{tenant.db_host} #{tenant.db_port} #{tenant.db_database} #{user.db_user}" + ) + + resp = + with {:ok, secret} <- H.get_user_secret(conn, tenant.auth_query, db_user) do + t = if secret.digest == :md5, do: :auth_query_md5, else: :auth_query + {:ok, {t, fn -> Map.put(secret, :alias, user.db_user_alias) end}} + end + + GenServer.stop(conn, :normal, 5_000) + Logger.info("ProxyClient: Get secrets finished") + resp + end + + @spec exchange_first(:password | :auth_query, fun(), binary(), binary(), binary()) :: + {binary(), map()} + def exchange_first(:password, secret, nonce, user, channel) do + message = Server.exchange_first_message(nonce) + server_first_parts = H.parse_server_first(message, nonce) + + {client_final_message, server_proof} = + H.get_client_final( + :password, + secret.().password, + server_first_parts, + nonce, + user, + channel + ) + + sings = %{ + client: List.last(client_final_message), + server: server_proof + } + + {message, sings} + end + + def exchange_first(:auth_query, secret, nonce, user, channel) do + secret = secret.() + message = Server.exchange_first_message(nonce, secret.salt) + server_first_parts = H.parse_server_first(message, nonce) + + sings = + H.signatures( + secret.stored_key, + secret.server_key, + server_first_parts, + nonce, + user, + channel + ) + + {message, sings} + end + + defp db_pid_meta({_, {_, pid}} = _key) do + rkey = Supavisor.Registry.PoolPids + fnode = node(pid) + + if fnode == node(), + do: Registry.lookup(rkey, pid), + else: :erpc.call(fnode, Registry, :lookup, [rkey, pid], 15_000) + end + + @spec timeout_check(atom, non_neg_integer) :: {:timeout, non_neg_integer, atom} + def timeout_check(key, timeout) do + {:timeout, timeout, key} + end + + @spec handle_actions(map) :: [{:timeout, non_neg_integer, atom}] + defp handle_actions(%{} = data) do + heartbeat = + if data.heartbeat_interval > 0, + do: [{:timeout, data.heartbeat_interval, :heartbeat_check}], + else: [] + + idle = + if data.idle_timeout > 0, do: [{:timeout, data.idle_timeout, :idle_timeout}], else: [] + + idle ++ heartbeat + end + + @spec app_name(any()) :: String.t() + def app_name(name) when is_binary(name) do + suffix = " via Supavisor" + # https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-APPLICATION-NAME + max_len = 64 + suffix_len = 14 + + if String.length(name) <= max_len - suffix_len do + name <> suffix + else + truncated_name = String.slice(name, 0, max_len - suffix_len - 3) + truncated_name <> "..." <> suffix + end + end + + def app_name(name) do + Logger.error("ProxyClient: Invalid application name #{inspect(name)}") + "via Supavisor" + end + + @spec maybe_change_log(map()) :: atom() | nil + def maybe_change_log(%{"payload" => %{"options" => options}}) do + level = options["log_level"] && String.to_existing_atom(options["log_level"]) + + if level in [:debug, :info, :notice, :warning, :error] do + H.set_log_level(level) + level + end + end + + def maybe_change_log(_), do: :ok +end diff --git a/lib/supavisor/handlers/proxy/db.ex b/lib/supavisor/handlers/proxy/db.ex new file mode 100644 index 00000000..c9d35c81 --- /dev/null +++ b/lib/supavisor/handlers/proxy/db.ex @@ -0,0 +1,260 @@ +defmodule Supavisor.Handlers.Proxy.Db do + @moduledoc false + + require Logger + + alias Supavisor, as: S + alias Supavisor.Helpers, as: H + alias Supavisor.HandlerHelpers, as: HH + alias Supavisor.{Monitoring.Telem, Protocol.Server} + + @type state :: :connect | :authentication | :idle | :busy + + @sock_closed [:tcp_closed, :ssl_closed] + @proto [:tcp, :ssl] + + def handle_event(:info, {proto, _, bin}, :db_authentication, data) when proto in @proto do + dec_pkt = Server.decode(bin) + Logger.debug("ProxyDb: dec_pkt, #{inspect(dec_pkt, pretty: true)}") + HH.active_once(data.db_sock) + + resp = Enum.reduce(dec_pkt, %{}, &handle_auth_pkts(&1, &2, data)) + + case resp do + {:authentication_sasl, nonce} -> + {:keep_state, %{data | nonce: nonce}} + + {:authentication_server_first_message, server_proof} -> + {:keep_state, %{data | server_proof: server_proof}} + + :authentication_md5 -> + {:keep_state, data} + + {:error_response, ["SFATAL", "VFATAL", "C28P01", reason, _, _, _]} -> + Logger.error("ProxyDb: Auth error #{inspect(reason)}") + {:stop, :invalid_password, data} + + {:error_response, error} -> + Logger.error("ProxyDb: Error auth response #{inspect(error)}") + {:keep_state, data} + + {:ready_for_query, acc} -> + ps = acc.ps + backend_key_data = acc.backend_key_data + msg = "ProxyDb: DB ready_for_query: #{inspect(acc.db_state)} #{inspect(ps, pretty: true)}" + Logger.debug(msg) + ps_encoded = Server.encode_parameter_status(ps) + + {:next_state, :idle, %{data | parameter_status: ps, backend_key_data: backend_key_data}, + {:next_event, :internal, {:client, {:greetings, ps_encoded}}}} + + other -> + Logger.error("ProxyDb: Undefined auth response #{inspect(other)}") + {:stop, :auth_error, data} + end + end + + # forwards the message to the client + def handle_event(:info, {proto, _, bin}, _, data) when proto in @proto do + HH.sock_send(data.sock, bin) + HH.active_once(data.db_sock) + + data = + if String.ends_with?(bin, Server.ready_for_query()) do + Logger.debug("ProxyDb: collected network usage") + {_, stats} = Telem.network_usage(:client, data.sock, data.id, data.stats) + {_, db_stats} = Telem.network_usage(:db, data.db_sock, data.id, data.db_stats) + %{data | stats: stats, db_stats: db_stats} + else + data + end + + {:keep_state, data} + end + + def handle_event(_, {closed, _}, state, data) when closed in @sock_closed do + Logger.error("ProxyDb: Connection closed when state was #{state}") + Telem.handler_action(:db_handler, :stopped, data.id) + HH.sock_send(data.sock, Server.error_message("XX000", "Database connection closed")) + HH.sock_close(data.sock) + {:stop, :db_socket_closed, data} + end + + ## Internal functions + + @spec handle_auth_pkts(map(), map(), map()) :: any() + defp handle_auth_pkts(%{tag: :parameter_status, payload: {k, v}}, acc, _), + do: update_in(acc, [:ps], fn ps -> Map.put(ps || %{}, k, v) end) + + defp handle_auth_pkts(%{tag: :ready_for_query, payload: db_state}, acc, _), + do: {:ready_for_query, Map.put(acc, :db_state, db_state)} + + defp handle_auth_pkts(%{tag: :backend_key_data, payload: payload}, acc, _), + do: Map.put(acc, :backend_key_data, payload) + + defp handle_auth_pkts(%{payload: {:authentication_sasl_password, methods_b}}, _, data) do + nonce = + case Server.decode_string(methods_b) do + {:ok, req_method, _} -> + Logger.debug("ProxyDb: SASL method #{inspect(req_method)}") + nonce = :pgo_scram.get_nonce(16) + user = get_user(data.auth) + client_first = :pgo_scram.get_client_first(user, nonce) + client_first_size = IO.iodata_length(client_first) + + sasl_initial_response = [ + "SCRAM-SHA-256", + 0, + <>, + client_first + ] + + bin = :pgo_protocol.encode_scram_response_message(sasl_initial_response) + :ok = HH.sock_send(data.db_sock, bin) + nonce + + other -> + Logger.error("ProxyDb: Undefined sasl method #{inspect(other)}") + nil + end + + {:authentication_sasl, nonce} + end + + defp handle_auth_pkts( + %{payload: {:authentication_server_first_message, server_first}}, + _, + data + ) + when data.auth.require_user == false do + nonce = data.nonce + server_first_parts = H.parse_server_first(server_first, nonce) + + {client_final_message, server_proof} = + H.get_client_final( + :auth_query, + data.auth.secrets.(), + server_first_parts, + nonce, + data.auth.secrets.().user, + "biws" + ) + + bin = :pgo_protocol.encode_scram_response_message(client_final_message) + :ok = HH.sock_send(data.db_sock, bin) + + {:authentication_server_first_message, server_proof} + end + + defp handle_auth_pkts( + %{payload: {:authentication_server_first_message, server_first}}, + _, + data + ) do + nonce = data.nonce + server_first_parts = :pgo_scram.parse_server_first(server_first, nonce) + + {client_final_message, server_proof} = + :pgo_scram.get_client_final( + server_first_parts, + nonce, + data.auth.user, + data.auth.password.() + ) + + bin = :pgo_protocol.encode_scram_response_message(client_final_message) + :ok = HH.sock_send(data.db_sock, bin) + + {:authentication_server_first_message, server_proof} + end + + defp handle_auth_pkts( + %{payload: {:authentication_server_final_message, _server_final}}, + acc, + _data + ), + do: acc + + defp handle_auth_pkts(%{payload: {:authentication_md5_password, salt}} = dec_pkt, _, data) do + Logger.debug("ProxyDb: dec_pkt, #{inspect(dec_pkt, pretty: true)}") + + digest = + if data.auth.method == :password do + H.md5([data.auth.password.(), data.auth.user]) + else + data.auth.secrets.().secret + end + + payload = ["md5", H.md5([digest, salt]), 0] + bin = [?p, <>, payload] + :ok = HH.sock_send(data.db_sock, bin) + :authentication_md5 + end + + defp handle_auth_pkts(%{tag: :error_response, payload: error}, _acc, _data), + do: {:error_response, error} + + defp handle_auth_pkts(_e, acc, _data), do: acc + + @spec try_ssl_handshake(S.tcp_sock(), map()) :: {:ok, S.sock()} | {:error, term()} + def try_ssl_handshake(sock, %{upstream_ssl: true} = auth) do + with :ok <- HH.sock_send(sock, Server.ssl_request()) do + ssl_recv(sock, auth) + end + end + + def try_ssl_handshake(sock, _), do: {:ok, sock} + + @spec ssl_recv(S.tcp_sock(), map) :: {:ok, S.ssl_sock()} | {:error, term} + def ssl_recv({:gen_tcp, sock} = s, auth) do + case :gen_tcp.recv(sock, 1, 15_000) do + {:ok, <>} -> ssl_connect(s, auth) + {:ok, <>} -> {:error, :ssl_not_available} + {:error, _} = error -> error + end + end + + @spec ssl_connect(S.tcp_sock(), map, pos_integer) :: {:ok, S.ssl_sock()} | {:error, term} + def ssl_connect({:gen_tcp, sock}, auth, timeout \\ 5000) do + opts = + case auth.upstream_verify do + :peer -> + [ + verify: :verify_peer, + cacerts: [auth.upstream_tls_ca], + # unclear behavior on pg14 + server_name_indication: auth.sni_host || auth.host, + customize_hostname_check: [{:match_fun, fn _, _ -> true end}] + ] + + :none -> + [verify: :verify_none] + end + + case :ssl.connect(sock, opts, timeout) do + {:ok, ssl_sock} -> {:ok, {:ssl, ssl_sock}} + {:error, reason} -> {:error, reason} + end + end + + @spec send_startup(S.sock(), map()) :: :ok | {:error, term} + def send_startup(sock, auth) do + user = get_user(auth) + + msg = + :pgo_protocol.encode_startup_message([ + {"user", user}, + {"database", auth.database}, + {"application_name", auth.application_name} + ]) + + HH.sock_send(sock, msg) + end + + @spec get_user(map) :: String.t() + def get_user(auth) do + if auth.require_user, + do: auth.secrets.().db_user, + else: auth.secrets.().user + end +end diff --git a/lib/supavisor/handlers/proxy/handler.ex b/lib/supavisor/handlers/proxy/handler.ex new file mode 100644 index 00000000..51d7601c --- /dev/null +++ b/lib/supavisor/handlers/proxy/handler.ex @@ -0,0 +1,154 @@ +defmodule Supavisor.Handlers.Proxy.Handler do + @moduledoc false + + require Logger + + @behaviour :ranch_protocol + @behaviour :gen_statem + + alias Supavisor.{ + Helpers, + Protocol.Server, + Monitoring.PromEx, + Handlers.Proxy.Db, + Handlers.Proxy.Client + } + + alias Supavisor.HandlerHelpers, as: HH + + @sock_closed [:tcp_closed, :ssl_closed] + @proto [:tcp, :ssl] + + @impl true + def start_link(ref, transport, opts) do + pid = :proc_lib.spawn_link(__MODULE__, :init, [ref, transport, opts]) + {:ok, pid} + end + + @impl true + def callback_mode, do: [:handle_event_function] + + @impl true + def init(_), do: :ignore + + def init(ref, trans, opts) do + Process.flag(:trap_exit, true) + Helpers.set_max_heap_size(90) + + {:ok, sock} = :ranch.handshake(ref) + :ok = trans.setopts(sock, active: true) + Logger.debug("ClientHandler is: #{inspect(self())}") + + data = %{ + id: nil, + sock: {:gen_tcp, sock}, + db_sock: {:gen_tcp, nil}, + trans: trans, + db_pid: nil, + tenant: nil, + user: nil, + pool: nil, + manager: nil, + query_start: nil, + timeout: nil, + ps: nil, + ssl: false, + auth_secrets: nil, + proxy_type: nil, + mode: opts.mode, + stats: %{}, + db_stats: %{}, + idle_timeout: 0, + db_name: nil, + last_query: nil, + heartbeat_interval: 0, + connection_start: System.monotonic_time(), + log_level: nil, + version: Application.spec(:supavisor, :vsn), + nonce: nil, + server_proof: nil, + parameter_status: %{}, + app_name: nil, + peer_ip: Helpers.peer_ip(sock), + auth: %{}, + backend_key_data: %{} + } + + :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) + end + + @impl true + def handle_event(e, {:client, _} = msg, state, data) do + Client.handle_event(e, msg, state, data) + end + + def handle_event(:timeout = e, msg, state, data) do + Client.handle_event(e, msg, state, data) + end + + def handle_event(event, {proto, sock, _payload} = msg, state, %{sock: {_, sock}} = data) + when proto in @proto do + Client.handle_event(event, msg, state, data) + end + + def handle_event(event, {proto, sock, _payload} = msg, state, %{db_sock: {_, sock}} = data) + when proto in @proto do + Db.handle_event(event, msg, state, data) + end + + def handle_event(event, {closed, sock} = msg, state, %{sock: {_, sock}} = data) + when closed in @sock_closed do + Client.handle_event(event, msg, state, data) + end + + def handle_event(event, {closed, sock} = msg, state, %{db_sock: {_, sock}} = data) + when closed in @sock_closed do + Db.handle_event(event, msg, state, data) + end + + def handle_event(type, content, state, data) do + msg = [ + {"type", type}, + {"content", content}, + {"state", state}, + {"data", data} + ] + + Logger.debug("ProxyHandler: Undefined msg: #{inspect(msg, pretty: true)}") + + :keep_state_and_data + end + + @impl true + def terminate({:shutdown, reason}, state, data) do + HH.sock_send(data.sock, Server.error_message("XX000", "#{inspect(reason)}")) + clean_up(data) + + Logger.info( + "ProxyHandler: Terminating with reason: #{inspect(reason)} when state was #{state}" + ) + + :ok + end + + def terminate(reason, state, data) do + clean_up(data) + + Logger.info( + "ProxyHandler: Terminating with reason: #{inspect(reason)} when state was #{state}" + ) + end + + ## Internal functions + + @spec clean_up(map()) :: any() + defp clean_up(data) do + HH.sock_close(data.sock) + HH.sock_close(data.db_sock) + + case Registry.lookup(Supavisor.Registry.TenantClients, data.id) do + clients when clients == [{self(), []}] or clients == [] -> PromEx.remove_metrics(data.id) + _ -> :ok + end + end +end diff --git a/lib/supavisor/helpers.ex b/lib/supavisor/helpers.ex index d64ec52a..b2bd9680 100644 --- a/lib/supavisor/helpers.ex +++ b/lib/supavisor/helpers.ex @@ -357,4 +357,12 @@ defmodule Supavisor.Helpers do Logger.notice("Setting log level to #{inspect(level)}") Logger.put_process_level(self(), level) end + + @spec peer_ip(:gen_tcp.socket()) :: String.t() + def peer_ip(socket) do + case :inet.peername(socket) do + {:ok, {ip, _port}} -> List.to_string(:inet.ntoa(ip)) + _error -> "undefined" + end + end end diff --git a/lib/supavisor/monitoring/prom_ex.ex b/lib/supavisor/monitoring/prom_ex.ex index d62788ca..8e3dc78d 100644 --- a/lib/supavisor/monitoring/prom_ex.ex +++ b/lib/supavisor/monitoring/prom_ex.ex @@ -30,7 +30,8 @@ defmodule Supavisor.Monitoring.PromEx do end @spec remove_metrics(S.id()) :: non_neg_integer - def remove_metrics({{type, tenant}, user, mode, db_name}) do + def remove_metrics({{type, tenant}, user, mode, db_name} = id) do + Logger.debug("Removing metrics for #{inspect(id)}") meta = %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} Supavisor.Monitoring.PromEx.Metrics From 57d2ca2714fa1de12fa11a6abaca60c809982d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Thu, 18 Jul 2024 14:05:40 +0200 Subject: [PATCH 25/97] chore: add Nix Flake to project (#400) * chore: add Nix Flake to project Currently it defines only development environment for Supavisor. In the future the plan is to add package definition as well as NixOS module to the Flake. This will simplify deployment on NixOS platforms as well as should help with spawning virtual machines with Supavisor running on them. * ft: add derivation for Supavisor --- flake.lock | 455 ++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 120 +++++++++++++ nix/package.nix | 51 ++++++ 3 files changed, 626 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/package.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..9722da48 --- /dev/null +++ b/flake.lock @@ -0,0 +1,455 @@ +{ + "nodes": { + "cachix": { + "inputs": { + "devenv": "devenv_2", + "flake-compat": [ + "devenv", + "flake-compat" + ], + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "pre-commit-hooks": [ + "devenv", + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1712055811, + "narHash": "sha256-7FcfMm5A/f02yyzuavJe06zLa9hcMHsagE28ADcmQvk=", + "owner": "cachix", + "repo": "cachix", + "rev": "02e38da89851ec7fec3356a5c04bc8349cae0e30", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "cachix", + "type": "github" + } + }, + "devenv": { + "inputs": { + "cachix": "cachix", + "flake-compat": "flake-compat_2", + "nix": "nix_2", + "nixpkgs": [ + "nixpkgs" + ], + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1720699919, + "narHash": "sha256-5DNu5bWJbh3ZX6fWI4gJDsHcZlYCPSM7pWNMfoza+hA=", + "owner": "cachix", + "repo": "devenv", + "rev": "55106de9d798923df979e67811f8e1eda960c219", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "devenv_2": { + "inputs": { + "flake-compat": [ + "devenv", + "cachix", + "flake-compat" + ], + "nix": "nix", + "nixpkgs": "nixpkgs", + "poetry2nix": "poetry2nix", + "pre-commit-hooks": [ + "devenv", + "cachix", + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1708704632, + "narHash": "sha256-w+dOIW60FKMaHI1q5714CSibk99JfYxm0CzTinYWr+Q=", + "owner": "cachix", + "repo": "devenv", + "rev": "2ee4450b0f4b95a1b90f2eb5ffea98b90e48c196", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "python-rewrite", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1689068808, + "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "devenv", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "nix": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "devenv", + "cachix", + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1712911606, + "narHash": "sha256-BGvBhepCufsjcUkXnEEXhEVjwdJAwPglCC2+bInc794=", + "owner": "domenkozar", + "repo": "nix", + "rev": "b24a9318ea3f3600c1e24b4a00691ee912d4de12", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "devenv-2.21", + "repo": "nix", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "devenv", + "cachix", + "devenv", + "poetry2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688870561, + "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nix_2": { + "inputs": { + "flake-compat": [ + "devenv", + "flake-compat" + ], + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression_2" + }, + "locked": { + "lastModified": 1712911606, + "narHash": "sha256-BGvBhepCufsjcUkXnEEXhEVjwdJAwPglCC2+bInc794=", + "owner": "domenkozar", + "repo": "nix", + "rev": "b24a9318ea3f3600c1e24b4a00691ee912d4de12", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "devenv-2.21", + "repo": "nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1692808169, + "narHash": "sha256-x9Opq06rIiwdwGeK2Ykj69dNc2IvUH1fY55Wm7atwrE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9201b5ff357e781bf014d0330d18555695df7ba8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1719876945, + "narHash": "sha256-Fm2rDDs86sHy0/1jxTOKB1118Q0O3Uc7EC0iXvXKpbI=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/5daf0514482af3f97abaefc78a6606365c9108e2.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/5daf0514482af3f97abaefc78a6606365c9108e2.tar.gz" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-regression_2": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1710695816, + "narHash": "sha256-3Eh7fhEID17pv9ZxrPwCLfqXnYP006RKzSs0JptsN84=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "614b4613980a522ba49f0d194531beddbb7220d3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1720594544, + "narHash": "sha256-w6dlBUQYvS65f0Z33TvkcAj7ITr4NFqhF5ywss5T5bU=", + "path": "/nix/store/3x6hmlqc6682q9z6n11ynvmq19hfrgn0-source", + "rev": "aa9461550594533c29866d42f861b6ff079a7fb6", + "type": "path" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "poetry2nix": { + "inputs": { + "flake-utils": "flake-utils", + "nix-github-actions": "nix-github-actions", + "nixpkgs": [ + "devenv", + "cachix", + "devenv", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1692876271, + "narHash": "sha256-IXfZEkI0Mal5y1jr6IRWMqK8GW2/f28xJenZIPQqkY0=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "d5006be9c2c2417dafb2e2e5034d83fabd207ee3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "poetry2nix", + "type": "github" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": [ + "devenv", + "flake-compat" + ], + "flake-utils": "flake-utils_2", + "gitignore": "gitignore", + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1713775815, + "narHash": "sha256-Wu9cdYTnGQQwtT20QQMg7jzkANKQjwBD9iccfGKkfls=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "2ac4dcbf55ed43f3be0bae15e181f08a57af24a4", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs_2" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..5483b445 --- /dev/null +++ b/flake.nix @@ -0,0 +1,120 @@ +{ + description = "Elixir's application"; + + inputs.nixpkgs.url = "flake:nixpkgs"; + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + + inputs.devenv = { + url = "github:cachix/devenv"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + outputs = { + self, + flake-parts, + devenv, + ... + } @ inputs: + flake-parts.lib.mkFlake {inherit inputs;} { + flake = {}; + + systems = [ + "x86_64-linux" + "x86_64-darwin" + "aarch64-linux" + "aarch64-darwin" + ]; + + perSystem = { + self', + inputs', + pkgs, + lib, + ... + }: { + formatter = pkgs.alejandra; + + apps.up = { + type = "app"; + program = toString self'.devShells.default.config.procfileScript; + }; + + packages = { + supavisor = let + erl = pkgs.beam_nox.packages.erlang_26; + in erl.callPackage ./nix/package.nix {}; + + default = self'.packages.supavisor; + }; + + devShells.default = devenv.lib.mkShell { + inherit inputs pkgs; + + modules = [ + { + languages.elixir = { + enable = true; + package = pkgs.beam.packages.erlang_26.elixir_1_17; + }; + packages = [ + pkgs.lexical + ]; + + # env.DYLD_INSERT_LIBRARIES = "${pkgs.mimalloc}/lib/libmimalloc.dylib"; + } + { + packages = [ + pkgs.pgbouncer + ]; + + services.postgres = { + enable = true; + initialScript = '' + ${builtins.readFile ./dev/postgres/00-setup.sql} + + CREATE USER postgres SUPERUSER PASSWORD 'postgres'; + ''; + listen_addresses = "127.0.0.1"; + port = 6432; + }; + + # Force connection through TCP instead of Unix socket + env.PGHOST = lib.mkForce ""; + } + ({ + pkgs, + lib, + config, + ... + }: { + languages.rust.enable = true; + languages.cplusplus.enable = true; + + packages = + [ + pkgs.protobuf + pkgs.cargo-outdated + ] + ++ lib.optionals pkgs.stdenv.isDarwin (with pkgs.darwin.apple_sdk; [ + frameworks.System + frameworks.CoreFoundation + frameworks.CoreServices + frameworks.DiskArbitration + frameworks.IOKit + frameworks.CFNetwork + frameworks.Security + libs.libDER + ]); + + # Workaround for https://github.com/rust-lang/cargo/issues/5376 + env.RUSTFLAGS = lib.mkForce (lib.optionals pkgs.stdenv.isDarwin [ + "-L framework=${config.devenv.profile}/Library/Frameworks" + "-C link-arg=-undefined" + "-C link-arg=dynamic_lookup" + ]); + }) + ]; + }; + }; + }; +} diff --git a/nix/package.nix b/nix/package.nix new file mode 100644 index 00000000..dcaa164d --- /dev/null +++ b/nix/package.nix @@ -0,0 +1,51 @@ +{ + fetchMixDeps, + mixRelease, + cargo, + rustPlatform, + lib, + stdenv, + darwin, + protobuf, + libiconv, +}: +let + pname = "supavisor"; + version = "0.0.1"; + src = ./..; + + mixFodDeps = fetchMixDeps { + pname = "mix-deps-${pname}"; + inherit src version; + hash = "sha256-vTBDNIZ6Pp23u70f8oTe3nbpReCEDPf6VuWNLdkWwq4="; + }; + + cargoDeps = rustPlatform.importCargoLock { + lockFile = ../native/pgparser/Cargo.lock; + }; +in mixRelease { + inherit pname version src mixFodDeps; + + nativeBuildInputs = [cargo protobuf]; + + buildInputs = lib.optionals stdenv.isDarwin (with darwin.apple_sdk; [ + libiconv + frameworks.System + frameworks.CoreFoundation + frameworks.CoreServices + frameworks.DiskArbitration + frameworks.IOKit + frameworks.CFNetwork + frameworks.Security + libs.libDER + ]); + + preConfigure = '' + cat ${cargoDeps}/.cargo/config >> native/pgparser/.cargo/config.toml + ln -s ${cargoDeps} native/pgparser/cargo-vendor-dir + ''; + + meta = { + mainProgram = "supavisor"; + }; +} From c7c93339c421e028c9c0cb404ce1a5dc6e069307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 23 Jul 2024 14:43:11 +0200 Subject: [PATCH 26/97] fix: correct Peep Git commit hash after force push (#404) --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 40e65256..ea112bc1 100644 --- a/mix.lock +++ b/mix.lock @@ -44,7 +44,7 @@ "open_api_spex": {:hex, :open_api_spex, "3.19.1", "65ccb5d06e3d664d1eec7c5ea2af2289bd2f37897094a74d7219fb03fc2b5994", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "392895827ce2984a3459c91a484e70708132d8c2c6c5363972b4b91d6bbac3dd"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.3.0", "03e2177f28dd8d11aaa88e8522c81c2f6a788170fe52f7a65262340961e663f9", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "b9e5ff775fd064fa098dba3c398490b77649a352b40b0b730a6b7dc0bdd68858"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, - "peep": {:git, "https://github.com/hauleth/peep.git", "7f640c06e3ee4ce4f4e5dd9f262cb3377dfcb1ce", [branch: "ft/custom-prometheus-types"]}, + "peep": {:git, "https://github.com/hauleth/peep.git", "4cad7e215ea173b235452fb805ede2867ac93d18", [branch: "ft/custom-prometheus-types"]}, "pg_types": {:hex, :pg_types, "0.4.0", "3ce365c92903c5bb59c0d56382d842c8c610c1b6f165e20c4b652c96fa7e9c14", [:rebar3], [], "hexpm", "b02efa785caececf9702c681c80a9ca12a39f9161a846ce17b01fb20aeeed7eb"}, "pgo": {:hex, :pgo, "0.14.0", "f53711d103d7565db6fc6061fcf4ff1007ab39892439be1bb02d9f686d7e6663", [:rebar3], [{:backoff, "~> 1.1.6", [hex: :backoff, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:pg_types, "~> 0.4.0", [hex: :pg_types, repo: "hexpm", optional: false]}], "hexpm", "71016c22599936e042dc0012ee4589d24c71427d266292f775ebf201d97df9c9"}, "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"}, From 4e6ec5f883879f9f404b15968ce66a381e5d0c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 30 Jul 2024 14:47:27 +0200 Subject: [PATCH 27/97] chore: use mainline Peep (#406) All changes required by our code are merged to mainline, so we do not need to use custom fork anymore. --- mix.exs | 3 +-- mix.lock | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index 6906eaa7..d4636aac 100644 --- a/mix.exs +++ b/mix.exs @@ -47,8 +47,7 @@ defmodule Supavisor.MixProject do {:phoenix_live_view, "~> 0.20.0"}, {:phoenix_live_dashboard, "~> 0.7"}, {:telemetry_poller, "~> 1.0"}, - # TODO: point it to Supabase fork of prom_ex when available - {:peep, github: "hauleth/peep", branch: "ft/custom-prometheus-types", override: true}, + {:peep, "~> 3.1"}, {:jason, "~> 1.2"}, {:plug_cowboy, "~> 2.5"}, {:joken, "~> 2.6.0"}, diff --git a/mix.lock b/mix.lock index ea112bc1..69db459e 100644 --- a/mix.lock +++ b/mix.lock @@ -44,7 +44,7 @@ "open_api_spex": {:hex, :open_api_spex, "3.19.1", "65ccb5d06e3d664d1eec7c5ea2af2289bd2f37897094a74d7219fb03fc2b5994", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "392895827ce2984a3459c91a484e70708132d8c2c6c5363972b4b91d6bbac3dd"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.3.0", "03e2177f28dd8d11aaa88e8522c81c2f6a788170fe52f7a65262340961e663f9", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "b9e5ff775fd064fa098dba3c398490b77649a352b40b0b730a6b7dc0bdd68858"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, - "peep": {:git, "https://github.com/hauleth/peep.git", "4cad7e215ea173b235452fb805ede2867ac93d18", [branch: "ft/custom-prometheus-types"]}, + "peep": {:hex, :peep, "3.1.0", "1680337d682dfde308b643814834379a5210eb11db5aaca8ea823c218d4d7e16", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "3de3c13f3efcff6130e53c8956d41c20a436650cfddee6e8aa6ebdef751d542f"}, "pg_types": {:hex, :pg_types, "0.4.0", "3ce365c92903c5bb59c0d56382d842c8c610c1b6f165e20c4b652c96fa7e9c14", [:rebar3], [], "hexpm", "b02efa785caececf9702c681c80a9ca12a39f9161a846ce17b01fb20aeeed7eb"}, "pgo": {:hex, :pgo, "0.14.0", "f53711d103d7565db6fc6061fcf4ff1007ab39892439be1bb02d9f686d7e6663", [:rebar3], [{:backoff, "~> 1.1.6", [hex: :backoff, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:pg_types, "~> 0.4.0", [hex: :pg_types, repo: "hexpm", optional: false]}], "hexpm", "71016c22599936e042dc0012ee4589d24c71427d266292f775ebf201d97df9c9"}, "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"}, @@ -61,7 +61,7 @@ "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "poolboy": {:git, "https://github.com/abc3/poolboy.git", "999ec7f5c7282d515020bb058b4832029d6d07bc", [tag: "v0.0.2"]}, "postgrex": {:hex, :postgrex, "0.18.0", "f34664101eaca11ff24481ed4c378492fed2ff416cd9b06c399e90f321867d7e", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a042989ba1bc1cca7383ebb9e461398e3f89f868c92ce6671feb7ef132a252d1"}, - "prom_ex": {:git, "https://github.com/hauleth/prom_ex.git", "34b714f8619c7637c728e3f385eb1cf1e75dc2fb", [branch: "ft/add-peep-storage"]}, + "prom_ex": {:git, "https://github.com/hauleth/prom_ex.git", "d6b41ffc6ba77a1dbbbf601eaa76af15a066c101", [branch: "ft/add-peep-storage"]}, "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, "recon": {:hex, :recon, "2.5.5", "c108a4c406fa301a529151a3bb53158cadc4064ec0c5f99b03ddb8c0e4281bdf", [:mix, :rebar3], [], "hexpm", "632a6f447df7ccc1a4a10bdcfce71514412b16660fe59deca0fcf0aa3c054404"}, "req": {:hex, :req, "0.5.2", "70b4976e5fbefe84e5a57fd3eea49d4e9aa0ac015301275490eafeaec380f97f", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0c63539ab4c2d6ced6114d2684276cef18ac185ee00674ee9af4b1febba1f986"}, From d554320a8e1e57cc2ff6ca67999dd672df7e017c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 30 Jul 2024 17:52:57 +0200 Subject: [PATCH 28/97] style(nix): reformat Nix files (#407) --- flake.nix | 13 +++++++------ nix/package.nix | 30 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/flake.nix b/flake.nix index 5483b445..0bf40e43 100644 --- a/flake.nix +++ b/flake.nix @@ -34,15 +34,14 @@ }: { formatter = pkgs.alejandra; - apps.up = { - type = "app"; - program = toString self'.devShells.default.config.procfileScript; - }; - packages = { + # Expose Devenv supervisor + devenv-up = self'.devShells.default.config.procfileScript; + supavisor = let erl = pkgs.beam_nox.packages.erlang_26; - in erl.callPackage ./nix/package.nix {}; + in + erl.callPackage ./nix/package.nix {}; default = self'.packages.supavisor; }; @@ -78,6 +77,8 @@ port = 6432; }; + process.implementation = "honcho"; + # Force connection through TCP instead of Unix socket env.PGHOST = lib.mkForce ""; } diff --git a/nix/package.nix b/nix/package.nix index dcaa164d..3bc4ae00 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -8,8 +8,7 @@ darwin, protobuf, libiconv, -}: -let +}: let pname = "supavisor"; version = "0.0.1"; src = ./..; @@ -23,12 +22,13 @@ let cargoDeps = rustPlatform.importCargoLock { lockFile = ../native/pgparser/Cargo.lock; }; -in mixRelease { - inherit pname version src mixFodDeps; +in + mixRelease { + inherit pname version src mixFodDeps; - nativeBuildInputs = [cargo protobuf]; + nativeBuildInputs = [cargo protobuf]; - buildInputs = lib.optionals stdenv.isDarwin (with darwin.apple_sdk; [ + buildInputs = lib.optionals stdenv.isDarwin (with darwin.apple_sdk; [ libiconv frameworks.System frameworks.CoreFoundation @@ -38,14 +38,14 @@ in mixRelease { frameworks.CFNetwork frameworks.Security libs.libDER - ]); + ]); - preConfigure = '' - cat ${cargoDeps}/.cargo/config >> native/pgparser/.cargo/config.toml - ln -s ${cargoDeps} native/pgparser/cargo-vendor-dir - ''; + preConfigure = '' + cat ${cargoDeps}/.cargo/config >> native/pgparser/.cargo/config.toml + ln -s ${cargoDeps} native/pgparser/cargo-vendor-dir + ''; - meta = { - mainProgram = "supavisor"; - }; -} + meta = { + mainProgram = "supavisor"; + }; + } From 52191dae8ce6a7fcff67ca87dc70ebe4b5e6f0c0 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 31 Jul 2024 15:57:51 +0200 Subject: [PATCH 29/97] feat: add new transaction handler (#402) --- Makefile | 40 ++- VERSION | 2 +- config/runtime.exs | 3 +- config/test.exs | 3 +- lib/supavisor.ex | 32 +++ lib/supavisor/application.ex | 9 +- lib/supavisor/db_handler.ex | 33 ++- lib/supavisor/handlers/proxy/client.ex | 266 +++++++++++++----- lib/supavisor/handlers/proxy/db.ex | 96 +++++-- lib/supavisor/handlers/proxy/handler.ex | 58 ++-- lib/supavisor/helpers.ex | 12 +- lib/supavisor/monitoring/tenant.ex | 1 + lib/supavisor/pg_parser.ex | 2 + lib/supavisor/protocol/client.ex | 3 + lib/supavisor/syn_handler.ex | 19 +- lib/supavisor/tenant_supervisor.ex | 9 + lib/supavisor/tenants/cluster.ex | 2 + lib/supavisor/tenants/cluster_tenants.ex | 2 + mix.lock | 4 +- priv/repo/seeds_after_migration.exs | 3 + test/integration/proxy_test.exs | 6 +- test/supavisor/syn_handler_test.exs | 3 +- .../controllers/metrics_controller_test.exs | 3 +- test/support/fixtures/single_connection.ex | 2 + 24 files changed, 458 insertions(+), 155 deletions(-) diff --git a/Makefile b/Makefile index 337fcfe8..36811b2f 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ dev.node2: CLUSTER_POSTGRES="true" \ PROXY_PORT_SESSION="5442" \ PROXY_PORT_TRANSACTION="6553" \ + NODE_IP=localhost \ ERL_AFLAGS="-kernel shell_history enabled" \ iex --name node2@127.0.0.1 --cookie cookie -S mix phx.server @@ -41,7 +42,7 @@ dev.node3: PROXY_PORT_SESSION="5443" \ PROXY_PORT_TRANSACTION="6554" \ ERL_AFLAGS="-kernel shell_history enabled" \ - iex --name node3@127.0.0.1 --cookie cookie -S mix phx.server + iex --name node3@127.0.0.1 --cookie cookie -S mix phx.server db_migrate: mix ecto.migrate --prefix _supavisor --log-migrator-sql @@ -100,5 +101,38 @@ dev_start_rel: FLY_ALLOC_ID=111e4567-e89b-12d3-a456-426614174000 \ SECRET_KEY_BASE="dev" \ CLUSTER_POSTGRES="true" \ - ERL_AFLAGS="-kernel shell_history enabled" \ - ./_build/dev/rel/supavisor/bin/supavisor start_iex + DB_POOL_SIZE="5" \ + _build/prod/rel/supavisor/bin/supavisor start_iex + +prod_rel: + rm -rf _build/prod && \ + MIX_ENV=prod mix compile && \ + MIX_ENV=prod mix release supavisor + +prod_start_rel: + MIX_ENV=prod \ + NODE_NAME=node1 \ + VAULT_ENC_KEY="aHD8DZRdk2emnkdktFZRh3E9RNg4aOY7" \ + API_JWT_SECRET=dev \ + METRICS_JWT_SECRET=dev \ + REGION=eu \ + FLY_ALLOC_ID=111e4567-e89b-12d3-a456-426614174000 \ + SECRET_KEY_BASE="dev" \ + CLUSTER_POSTGRES="true" \ + DB_POOL_SIZE="5" \ + _build/prod/rel/supavisor/bin/supavisor start_iex + +prod_start_rel2: + MIX_ENV=prod \ + NODE_NAME=node2 \ + PORT=4001 \ + VAULT_ENC_KEY="aHD8DZRdk2emnkdktFZRh3E9RNg4aOY7" \ + API_JWT_SECRET=dev \ + METRICS_JWT_SECRET=dev \ + REGION=eu \ + SECRET_KEY_BASE="dev" \ + CLUSTER_POSTGRES="true" \ + PROXY_PORT_SESSION="5442" \ + PROXY_PORT_TRANSACTION="6553" \ + NODE_IP=localhost \ + _build/prod/rel/supavisor/bin/supavisor start_iex diff --git a/VERSION b/VERSION index 03f7611d..227cea21 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.68 +2.0.0 diff --git a/config/runtime.exs b/config/runtime.exs index 57089160..cf0bf381 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -157,7 +157,8 @@ if config_env() != :test do global_downstream_key: downstream_key, reconnect_on_db_close: System.get_env("RECONNECT_ON_DB_CLOSE") == "true", api_blocklist: System.get_env("API_TOKEN_BLOCKLIST", "") |> String.split(","), - metrics_blocklist: System.get_env("METRICS_TOKEN_BLOCKLIST", "") |> String.split(",") + metrics_blocklist: System.get_env("METRICS_TOKEN_BLOCKLIST", "") |> String.split(","), + node_host: System.get_env("NODE_IP", "127.0.0.1") config :supavisor, Supavisor.Repo, url: System.get_env("DATABASE_URL", "ecto://postgres:postgres@localhost:6432/postgres"), diff --git a/config/test.exs b/config/test.exs index 13878d03..0f7fa4b4 100644 --- a/config/test.exs +++ b/config/test.exs @@ -14,7 +14,8 @@ config :supavisor, api_blocklist: [ "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJvbGUiOiJibG9ja2VkIiwiaWF0IjoxNjQ1MTkyODI0LCJleHAiOjE5NjA3Njg4MjR9.y-V3D1N2e8UTXc5PJzmV9cqMteq0ph2wl0yt42akQgA" ], - metrics_blocklist: [] + metrics_blocklist: [], + node_host: System.get_env("NODE_IP", "127.0.0.1") config :supavisor, Supavisor.Repo, username: "postgres", diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 169a0b1f..76fb517e 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -349,4 +349,36 @@ defmodule Supavisor do pid -> Manager.set_parameter_status(pid, ps) end end + + @spec get_pool_ranch(id) :: {:ok, map()} | {:error, :not_found} + def get_pool_ranch(id) do + case :syn.lookup(:tenants, id) do + {_sup_pid, %{port: _port, host: _host} = meta} -> {:ok, meta} + _ -> {:error, :not_found} + end + end + + @spec start_local_server(map()) :: {:ok, map()} | {:error, any()} + def start_local_server(%{max_clients: max_clients} = args) do + # max_clients=-1 is used for testing the maximum allowed clients in ProxyTest + {acceptors, max_clients} = + if max_clients > 0, + do: {ceil(max_clients / 100), max_clients}, + else: {1, 100} + + opts = + %{ + max_connections: max_clients, + num_acceptors: max(acceptors, 10), + socket_opts: [port: 0, keepalive: true] + } + + handler = Supavisor.Handlers.Proxy.Handler + args = Map.put(args, :local, true) + + with {:ok, pid} <- :ranch.start_listener(args.id, :ranch_tcp, opts, handler, args) do + host = Application.get_env(:supavisor, :node_host) + {:ok, %{listener: pid, host: host, port: :ranch.get_port(args.id)}} + end + end end diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index 760ef2ef..7e754862 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -6,7 +6,6 @@ defmodule Supavisor.Application do use Application require Logger alias Supavisor.Monitoring.PromEx - alias Supavisor.ClientHandler alias Supavisor.Handlers.Proxy.Handler, as: ProxyHandler @impl true @@ -31,9 +30,9 @@ defmodule Supavisor.Application do proxy_ports = [ {:pg_proxy_transaction, Application.get_env(:supavisor, :proxy_port_transaction), - :transaction, ClientHandler}, + :transaction, ProxyHandler}, {:pg_proxy_session, Application.get_env(:supavisor, :proxy_port_session), :session, - ProxyHandler} + Supavisor.ClientHandler} ] for {key, port, mode, handler} <- proxy_ports do @@ -41,9 +40,9 @@ defmodule Supavisor.Application do key, :ranch_tcp, %{ - max_connections: String.to_integer(System.get_env("MAX_CONNECTIONS") || "25000"), + max_connections: String.to_integer(System.get_env("MAX_CONNECTIONS") || "75000"), num_acceptors: String.to_integer(System.get_env("NUM_ACCEPTORS") || "100"), - socket_opts: [inet_backend: :socket, port: port, keepalive: true] + socket_opts: [port: port, keepalive: true] }, handler, %{mode: mode} diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 2c105c3d..2af17bae 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -31,6 +31,12 @@ defmodule Supavisor.DbHandler do @spec cast(pid(), pid(), binary()) :: :ok | {:error, any()} | {:buffering, non_neg_integer()} def cast(pid, caller, msg), do: :gen_statem.cast(pid, {:db_cast, caller, msg}) + @spec set_idle(pid()) :: :ok + def set_idle(pid), do: :gen_statem.cast(pid, :set_idle) + + @spec change_socket_owner(pid(), pid()) :: {:ok, S.sock()} + def change_socket_owner(pid, caller), do: :gen_statem.call(pid, {:tcp_owner, caller}, 15_000) + @spec get_state_and_mode(pid()) :: {:ok, {state, Supavisor.mode()}} | {:error, term()} def get_state_and_mode(pid) do try do @@ -69,7 +75,8 @@ defmodule Supavisor.DbHandler do server_proof: nil, stats: %{}, mode: args.mode, - replica_type: args.replica_type + replica_type: args.replica_type, + reply: nil } Telem.handler_action(:db_handler, :started, args.id) @@ -288,6 +295,12 @@ defmodule Supavisor.DbHandler do end end + def handle_event(:internal, :check_buffer, :idle, %{reply: {from, pid}} = data) do + :ok = H.controlling_process(data.sock, pid) + reply = {:reply, from, {:ok, data.sock}} + {:next_state, :busy, %{data | reply: nil}, reply} + end + def handle_event(:internal, :check_buffer, :idle, %{buffer: buff, caller: caller} = data) when is_pid(caller) do if buff != [] do @@ -391,6 +404,17 @@ defmodule Supavisor.DbHandler do {:next_event, :internal, :check_anon_buffer}} end + def handle_event({:call, from}, {:tcp_owner, pid}, state, %{sock: sock} = data) do + if state in [:idle, :busy] do + :ok = H.controlling_process(data.sock, pid) + reply = {:reply, from, {:ok, sock}} + {:keep_state, data, reply} + else + Logger.debug("DbHandler: TCP owner call when state was #{state}") + {:keep_state, %{data | reply: {from, pid}}} + end + end + def handle_event({:call, from}, {:db_call, caller, bin}, :idle, %{sock: sock} = data) do reply = {:reply, from, sock_send(sock, bin)} {:next_state, :busy, %{data | caller: caller}, reply} @@ -431,6 +455,11 @@ defmodule Supavisor.DbHandler do {:keep_state, %{data | caller: caller, buffer: new_buff}} end + def handle_event(:cast, :set_idle, _, data) do + Logger.debug("DbHandler: set_idle") + {:next_state, :idle, data} + end + def handle_event(_, {closed, _}, :busy, data) when closed in @sock_closed do {:stop, :db_termination, data} end @@ -452,7 +481,7 @@ defmodule Supavisor.DbHandler do end if state == :busy or data.mode == :session do - :ok = sock_send(data.sock, <>) + sock_send(data.sock, <>) :gen_tcp.close(elem(data.sock, 1)) {:stop, {:client_handler_down, data.mode}} else diff --git a/lib/supavisor/handlers/proxy/client.ex b/lib/supavisor/handlers/proxy/client.ex index 7a157fdf..72c12037 100644 --- a/lib/supavisor/handlers/proxy/client.ex +++ b/lib/supavisor/handlers/proxy/client.ex @@ -3,43 +3,60 @@ defmodule Supavisor.Handlers.Proxy.Client do require Logger - alias Supavisor, as: S - alias Supavisor.ProxyDb, as: Db - alias Supavisor.Helpers, as: H - alias Supavisor.HandlerHelpers, as: HH - alias Supavisor.{ Tenants, - ProxyHandlerDb, + Helpers, + DbHandler, + HandlerHelpers, + Protocol.Server, Monitoring.Telem, - Protocol.Client, - Protocol.Server + Handlers.Proxy.Db } - alias Supavisor.Handlers.Proxy.Db, as: ProxyDb - @cancel_query_msg <<16::32, 1234::16, 5678::16>> @sock_closed [:tcp_closed, :ssl_closed] @proto [:tcp, :ssl] def handle_event(:info, {_proto, _, <<"GET", _::binary>>}, :exchange, data) do Logger.debug("ProxyClient: Client is trying to request HTTP") - HH.sock_send(data.sock, "HTTP/1.1 204 OK\r\nx-app-version: #{data.version}\r\n\r\n") + + HandlerHelpers.sock_send( + data.sock, + "HTTP/1.1 204 OK\r\nx-app-version: #{data.version}\r\n\r\n" + ) + {:stop, {:shutdown, :http_request}} end # cancel request def handle_event(:info, {_, _, <<@cancel_query_msg, pid::32, key::32>>}, _, _) do Logger.debug("ProxyClient: Got cancel query for #{inspect({pid, key})}") - :ok = HH.send_cancel_query(pid, key, {:client, :cancel_query}) + :ok = HandlerHelpers.send_cancel_query(pid, key, {:client, :cancel_query}) {:stop, {:shutdown, :cancel_query}} end + def handle_event(:info, {:client, :cancel_query}, _, %{db_pid: db_pid}) + when is_pid(db_pid) do + Logger.debug("ProxyClient: Cancel query for #{inspect(db_pid)}") + + case db_pid_meta(db_pid) do + [{^db_pid, meta}] -> + :ok = HandlerHelpers.cancel_query(meta.host, meta.port, meta.ip_ver, meta.pid, meta.key) + + error -> + Logger.error( + "ClientHandler: Received cancel but no proc was found #{inspect(db_pid)} #{inspect(error)}" + ) + end + + :keep_state_and_data + end + def handle_event(:info, {:client, :cancel_query}, _, %{ auth: auth, backend_key_data: b }) do - :ok = HH.cancel_query(~c"#{auth.host}", auth.port, auth.ip_ver, b.pid, b.key) + :ok = HandlerHelpers.cancel_query(~c"#{auth.host}", auth.port, auth.ip_ver, b.pid, b.key) :keep_state_and_data end @@ -47,13 +64,13 @@ defmodule Supavisor.Handlers.Proxy.Client do def handle_event(:info, {:tcp, _, <<_::64>>}, :exchange, %{sock: sock} = data) do Logger.debug("ProxyClient: Client is trying to connect with SSL") - downstream_cert = H.downstream_cert() - downstream_key = H.downstream_key() + downstream_cert = Helpers.downstream_cert() + downstream_key = Helpers.downstream_key() # SSL negotiation, S/N/Error if !!downstream_cert and !!downstream_key do - :ok = HH.setopts(sock, active: false) - :ok = HH.sock_send(sock, "S") + :ok = HandlerHelpers.setopts(sock, active: false) + :ok = HandlerHelpers.sock_send(sock, "S") opts = [ certfile: downstream_cert, @@ -63,7 +80,7 @@ defmodule Supavisor.Handlers.Proxy.Client do case :ssl.handshake(elem(sock, 1), opts) do {:ok, ssl_sock} -> socket = {:ssl, ssl_sock} - :ok = HH.setopts(socket, active: true) + :ok = HandlerHelpers.setopts(socket, active: true) {:keep_state, %{data | sock: socket, ssl: true}} error -> @@ -74,7 +91,7 @@ defmodule Supavisor.Handlers.Proxy.Client do else Logger.error("ProxyClient: User requested SSL connection but no downstream cert/key found") - :ok = HH.sock_send(data.sock, "N") + :ok = HandlerHelpers.sock_send(data.sock, "N") :keep_state_and_data end end @@ -83,7 +100,7 @@ defmodule Supavisor.Handlers.Proxy.Client do case Server.decode_startup_packet(bin) do {:ok, hello} -> Logger.debug("ProxyClient: Client startup message: #{inspect(hello)}") - {type, {user, tenant_or_alias, db_name}} = HH.parse_user_info(hello.payload) + {type, {user, tenant_or_alias, db_name}} = HandlerHelpers.parse_user_info(hello.payload) # Validate user and db_name according to PostgreSQL rules. # The rules are: 1-63 characters, alphanumeric, underscore and $ @@ -101,7 +118,13 @@ defmodule Supavisor.Handlers.Proxy.Client do reason = "Invalid format for user or db_name" Logger.error("ProxyClient: #{inspect(reason)}") Telem.client_join(:fail, tenant_or_alias) - HH.send_error(data.sock, "XX000", "Authentication error, reason: #{inspect(reason)}") + + HandlerHelpers.send_error( + data.sock, + "XX000", + "Authentication error, reason: #{inspect(reason)}" + ) + {:stop, {:shutdown, :invalid_format}} end @@ -118,7 +141,7 @@ defmodule Supavisor.Handlers.Proxy.Client do :exchange, %{sock: sock} = data ) do - sni_hostname = HH.try_get_sni(sock) + sni_hostname = HandlerHelpers.try_get_sni(sock) case Tenants.get_user_cache(type, user, tenant_or_alias, sni_hostname) do {:ok, info} -> @@ -133,7 +156,7 @@ defmodule Supavisor.Handlers.Proxy.Client do db_name ) - mode = S.mode(id) + mode = Supavisor.mode(id) Logger.metadata( project: tenant_or_alias, @@ -147,22 +170,22 @@ defmodule Supavisor.Handlers.Proxy.Client do Registry.register(Supavisor.Registry.TenantClients, id, []) - {:ok, addr} = HH.addr_from_sock(sock) + {:ok, addr} = HandlerHelpers.addr_from_sock(sock) cond do - info.tenant.enforce_ssl and !data.ssl -> + info.tenant.enforce_ssl and !data.ssl and !data.local -> Logger.error( "ProxyClient: Tenant is not allowed to connect without SSL, user #{user}" ) - :ok = HH.send_error(sock, "XX000", "SSL connection is required") + :ok = HandlerHelpers.send_error(sock, "XX000", "SSL connection is required") Telem.client_join(:fail, id) {:stop, {:shutdown, :ssl_required}} - HH.filter_cidrs(info.tenant.allow_list, addr) == [] -> + HandlerHelpers.filter_cidrs(info.tenant.allow_list, addr) == [] -> message = "Address not in tenant allow_list: " <> inspect(addr) Logger.error("ProxyClient: #{message}") - :ok = HH.send_error(sock, "XX000", message) + :ok = HandlerHelpers.send_error(sock, "XX000", message) Telem.client_join(:fail, id) {:stop, {:shutdown, :address_not_allowed}} @@ -183,7 +206,11 @@ defmodule Supavisor.Handlers.Proxy.Client do Logger.error("ProxyClient: Authentication auth_secrets error: #{inspect(reason)}") :ok = - HH.send_error(sock, "XX000", "Authentication error, reason: #{inspect(reason)}") + HandlerHelpers.send_error( + sock, + "XX000", + "Authentication error, reason: #{inspect(reason)}" + ) Telem.client_join(:fail, id) {:stop, {:shutdown, :auth_secrets_error}} @@ -195,7 +222,7 @@ defmodule Supavisor.Handlers.Proxy.Client do "ProxyClient: User not found: #{inspect(reason)} #{inspect({type, user, tenant_or_alias})}" ) - :ok = HH.send_error(sock, "XX000", "Tenant or user not found") + :ok = HandlerHelpers.send_error(sock, "XX000", "Tenant or user not found") Telem.client_join(:fail, data.id) {:stop, {:shutdown, :user_not_found}} end @@ -247,7 +274,7 @@ defmodule Supavisor.Handlers.Proxy.Client do Logger.debug("ProxyClient: Cache hit for #{inspect(key)}") end - HH.sock_send(sock, msg) + HandlerHelpers.sock_send(sock, msg) Telem.client_join(:fail, data.id) {:stop, {:shutdown, :exchange_error}} @@ -262,20 +289,75 @@ defmodule Supavisor.Handlers.Proxy.Client do end Logger.debug("ProxyClient: Exchange success") - :ok = HH.sock_send(sock, Server.authentication_ok()) + :ok = HandlerHelpers.sock_send(sock, Server.authentication_ok()) Telem.client_join(:ok, data.id) auth = Map.merge(data.auth, %{secrets: secrets, method: method}) + conn_type = + case data.mode do + :transaction -> :subscribe + :session -> :connect_db + end + {:keep_state, %{data | auth_secrets: {method, secrets}, auth: auth}, - {:next_event, :internal, {:client, :connect_db}}} + {:next_event, :internal, {:client, conn_type}}} + end + end + + def handle_event(:internal, {:client, :subscribe}, _, data) do + Logger.warning("ClientHandler: Subscribe to tenant #{inspect(data.id)}") + + with {:ok, sup} <- + Supavisor.start_dist(data.id, data.auth_secrets, log_level: data.log_level), + true <- if(node(sup) == node(), do: true, else: :proxy), + {:ok, opts} <- Supavisor.subscribe(sup, data.id) do + Process.monitor(opts.workers.manager) + data = Map.merge(data, opts.workers) + data = %{data | idle_timeout: opts.idle_timeout} + + next = + if opts.ps == [] do + {:timeout, 10_000, :wait_ps} + else + {:next_event, :internal, {:client, {:greetings, opts.ps}}} + end + + {:keep_state, data, next} + else + {:error, :max_clients_reached} -> + msg = "Max client connections reached" + Logger.error("ClientHandler: #{msg}") + :ok = HandlerHelpers.send_error(data.sock, "XX000", msg) + Telem.client_join(:fail, data.id) + {:stop, {:shutdown, :max_clients_reached}} + + :proxy -> + {:ok, %{port: port, host: host}} = Supavisor.get_pool_ranch(data.id) + + auth = + Map.merge(data.auth, %{ + port: port, + host: host, + ip_version: :v4, + upstream_ssl: false, + upstream_tls_ca: nil, + upstream_verify: nil + }) + + data = Map.merge(data, %{auth: auth, mode: :session, proxy: true}) + {:keep_state, data, {:next_event, :internal, {:client, :connect_db}}} + + error -> + Logger.error("ClientHandler: Subscribe error: #{inspect(error)}") + {:keep_state_and_data, {:timeout, 1000, :subscribe}} end end def handle_event(:internal, {:client, :connect_db}, _, %{auth: auth} = data) do Logger.debug("Try to connect to DB") Telem.handler_action(:db_handler, :db_connection, data.id) - ip_ver = H.ip_version(auth.ip_version, auth.host) + ip_ver = Helpers.ip_version(auth.ip_version, auth.host) sock_opts = [ :binary, @@ -287,13 +369,15 @@ defmodule Supavisor.Handlers.Proxy.Client do case :gen_tcp.connect(~c"#{auth.host}", auth.port, sock_opts) do {:ok, sock} -> - Logger.debug("ProxyClient: auth #{inspect(auth, pretty: true)}") + Logger.debug("ProxyClient: auth #{inspect(data, pretty: true)}") - case ProxyDb.try_ssl_handshake({:gen_tcp, sock}, auth) do + case Db.try_ssl_handshake({:gen_tcp, sock}, auth) do {:ok, sock} -> - case ProxyDb.send_startup(sock, auth) do + tenant = if(data.proxy, do: Supavisor.tenant(data.id)) + + case Db.send_startup(sock, auth, tenant) do :ok -> - HH.active_once(sock) + HandlerHelpers.active_once(sock) auth = Map.put(auth, :ip_ver, ip_ver) {:next_state, :db_authentication, %{data | db_sock: sock, auth: auth}} @@ -319,9 +403,9 @@ defmodule Supavisor.Handlers.Proxy.Client do def handle_event(:internal, {:client, {:greetings, ps}}, _, %{sock: sock} = data) do {header, <> = payload} = Server.backend_key_data() msg = [ps, [header, payload], Server.ready_for_query()] - :ok = HH.listen_cancel_query(pid, key) - :ok = HH.sock_send(sock, msg) - HH.active_once(sock) + :ok = HandlerHelpers.listen_cancel_query(pid, key) + :ok = HandlerHelpers.sock_send(sock, msg) + HandlerHelpers.active_once(sock) Telem.client_connection_time(data.connection_start, data.id) {:next_state, :idle, data, handle_actions(data)} end @@ -344,14 +428,41 @@ defmodule Supavisor.Handlers.Proxy.Client do def handle_event(:timeout, :heartbeat_check, _, data) do Logger.debug("ProxyClient: Send heartbeat to client") - HH.sock_send(data.sock, Server.application_name()) + HandlerHelpers.sock_send(data.sock, Server.application_name()) {:keep_state_and_data, {:timeout, data.heartbeat_interval, :heartbeat_check}} end + def handle_event( + :info, + {proto, _, <>}, + _, + %{mode: :transaction, db_pid: nil} + ) + when proto in @proto do + {:stop, {:shutdown, :terminate_received}} + end + # forwards the message to the db + def handle_event( + :info, + {proto, _, bin}, + _, + %{mode: :transaction, db_pid: nil} = data + ) + when proto in @proto do + {time, {db_pid, db_sock}} = :timer.tc(__MODULE__, :checkout, [data.pool, data.timeout]) + same_box = if node(db_pid) == node(), do: :local, else: :remote + Telem.pool_checkout_time(time, data.id, same_box) + Logger.debug("ProxyClient: Checkout new db connection #{inspect({db_pid, db_sock})}") + + HandlerHelpers.sock_send(db_sock, bin) + HandlerHelpers.active_once(data.sock) + {:keep_state, %{data | db_pid: db_pid, db_sock: db_sock}} + end + def handle_event(:info, {proto, _, bin}, _, data) when proto in @proto do - HH.sock_send(data.db_sock, bin) - HH.active_once(data.sock) + HandlerHelpers.sock_send(data.db_sock, bin) + HandlerHelpers.active_once(data.sock) :keep_state_and_data end @@ -360,17 +471,18 @@ defmodule Supavisor.Handlers.Proxy.Client do # client closed connection def handle_event(_, {closed, _}, _, data) - when closed in [:tcp_closed, :ssl_closed] do + when closed in @sock_closed do Logger.debug("ProxyClient: #{closed} socket closed for #{inspect(data.tenant)}") {:stop, {:shutdown, :socket_closed}} end ## Internal functions - @spec handle_exchange(S.sock(), {atom(), fun()}) :: {:ok, binary() | nil} | {:error, String.t()} + @spec handle_exchange(Supavisor.sock(), {atom(), fun()}) :: + {:ok, binary() | nil} | {:error, String.t()} def handle_exchange({_, socket} = sock, {:auth_query_md5 = method, secrets}) do salt = :crypto.strong_rand_bytes(4) - :ok = HH.sock_send(sock, Server.md5_request(salt)) + :ok = HandlerHelpers.sock_send(sock, Server.md5_request(salt)) with {:ok, %{ @@ -386,7 +498,7 @@ defmodule Supavisor.Handlers.Proxy.Client do end def handle_exchange({_, socket} = sock, {method, secrets}) do - :ok = HH.sock_send(sock, Server.scram_request()) + :ok = HandlerHelpers.sock_send(sock, Server.scram_request()) with {:ok, %{ @@ -411,7 +523,7 @@ defmodule Supavisor.Handlers.Proxy.Client do ), {:ok, key} <- authenticate_exchange(method, secrets, signatures, p) do message = "v=#{Base.encode64(signatures.server)}" - :ok = HH.sock_send(sock, Server.exchange_message(:final, message)) + :ok = HandlerHelpers.sock_send(sock, Server.exchange_message(:final, message)) {:ok, key} else {:error, message} -> {:error, message} @@ -430,7 +542,7 @@ defmodule Supavisor.Handlers.Proxy.Client do def reply_first_exchange(sock, method, secrets, channel, nonce, user) do {message, signatures} = exchange_first(method, secrets, nonce, user, channel) - :ok = HH.sock_send(sock, Server.exchange_message(:first, message)) + :ok = HandlerHelpers.sock_send(sock, Server.exchange_message(:first, message)) {:ok, signatures} end @@ -443,18 +555,19 @@ defmodule Supavisor.Handlers.Proxy.Client do def authenticate_exchange(:auth_query, secrets, signatures, p) do client_key = :crypto.exor(Base.decode64!(p), signatures.client) - if H.hash(client_key) == secrets.().stored_key, + if Helpers.hash(client_key) == secrets.().stored_key, do: {:ok, client_key}, else: {:error, "Wrong password"} end def authenticate_exchange(:auth_query_md5, client_hash, server_hash, salt) do - if "md5" <> H.md5([server_hash, salt]) == client_hash, + if "md5" <> Helpers.md5([server_hash, salt]) == client_hash, do: {:ok, nil}, else: {:error, "Wrong password"} end - @spec update_user_data(map(), map(), String.t(), S.id(), String.t(), S.mode()) :: map() + @spec update_user_data(map(), map(), String.t(), Supavisor.id(), String.t(), Supavisor.mode()) :: + map() def update_user_data(data, info, user, id, db_name, mode) do proxy_type = if info.tenant.require_user, do: :password, else: :auth_query @@ -489,7 +602,7 @@ defmodule Supavisor.Handlers.Proxy.Client do end @spec auth_secrets(map, String.t(), term(), non_neg_integer()) :: - {:ok, S.secrets()} | {:error, term()} + {:ok, Supavisor.secrets()} | {:error, term()} ## password secrets def auth_secrets(%{user: user, tenant: %{require_user: true}}, _, _, _) do secrets = %{db_user: user.db_user, password: user.db_password, alias: user.db_user_alias} @@ -515,10 +628,10 @@ defmodule Supavisor.Handlers.Proxy.Client do @spec get_secrets(map, String.t()) :: {:ok, {:auth_query, fun()}} | {:error, term()} def get_secrets(%{user: user, tenant: tenant}, db_user) do ssl_opts = - if tenant.upstream_ssl and tenant.upstream_verify == "peer" do + if tenant.upstream_ssl and tenant.upstream_verify == :peer do [ {:verify, :verify_peer}, - {:cacerts, [H.upstream_cert(tenant.upstream_tls_ca)]}, + {:cacerts, [Helpers.upstream_cert(tenant.upstream_tls_ca)]}, {:server_name_indication, String.to_charlist(tenant.db_host)}, {:customize_hostname_check, [{:match_fun, fn _, _ -> true end}]} ] @@ -534,7 +647,7 @@ defmodule Supavisor.Handlers.Proxy.Client do parameters: [application_name: "Supavisor auth_query"], ssl: tenant.upstream_ssl, socket_options: [ - H.ip_version(tenant.ip_version, tenant.db_host) + Helpers.ip_version(tenant.ip_version, tenant.db_host) ], queue_target: 1_000, queue_interval: 5_000, @@ -549,7 +662,7 @@ defmodule Supavisor.Handlers.Proxy.Client do ) resp = - with {:ok, secret} <- H.get_user_secret(conn, tenant.auth_query, db_user) do + with {:ok, secret} <- Helpers.get_user_secret(conn, tenant.auth_query, db_user) do t = if secret.digest == :md5, do: :auth_query_md5, else: :auth_query {:ok, {t, fn -> Map.put(secret, :alias, user.db_user_alias) end}} end @@ -563,10 +676,10 @@ defmodule Supavisor.Handlers.Proxy.Client do {binary(), map()} def exchange_first(:password, secret, nonce, user, channel) do message = Server.exchange_first_message(nonce) - server_first_parts = H.parse_server_first(message, nonce) + server_first_parts = Helpers.parse_server_first(message, nonce) {client_final_message, server_proof} = - H.get_client_final( + Helpers.get_client_final( :password, secret.().password, server_first_parts, @@ -586,10 +699,10 @@ defmodule Supavisor.Handlers.Proxy.Client do def exchange_first(:auth_query, secret, nonce, user, channel) do secret = secret.() message = Server.exchange_first_message(nonce, secret.salt) - server_first_parts = H.parse_server_first(message, nonce) + server_first_parts = Helpers.parse_server_first(message, nonce) sings = - H.signatures( + Helpers.signatures( secret.stored_key, secret.server_key, server_first_parts, @@ -601,15 +714,6 @@ defmodule Supavisor.Handlers.Proxy.Client do {message, sings} end - defp db_pid_meta({_, {_, pid}} = _key) do - rkey = Supavisor.Registry.PoolPids - fnode = node(pid) - - if fnode == node(), - do: Registry.lookup(rkey, pid), - else: :erpc.call(fnode, Registry, :lookup, [rkey, pid], 15_000) - end - @spec timeout_check(atom, non_neg_integer) :: {:timeout, non_neg_integer, atom} def timeout_check(key, timeout) do {:timeout, timeout, key} @@ -653,10 +757,28 @@ defmodule Supavisor.Handlers.Proxy.Client do level = options["log_level"] && String.to_existing_atom(options["log_level"]) if level in [:debug, :info, :notice, :warning, :error] do - H.set_log_level(level) + Helpers.set_log_level(level) level end end def maybe_change_log(_), do: :ok + + @spec checkout(pid(), non_neg_integer()) :: {pid(), Supavisor.sock()} + def checkout(pool, timeout) do + db_pid = :poolboy.checkout(pool, true, timeout) + Process.link(db_pid) + {:ok, db_sock} = DbHandler.change_socket_owner(db_pid, self()) + {db_pid, db_sock} + end + + @spec db_pid_meta(pid()) :: [{pid(), map()}] + defp db_pid_meta(pid) do + rkey = Supavisor.Registry.PoolPids + fnode = node(pid) + + if fnode == node(), + do: Registry.lookup(rkey, pid), + else: :erpc.call(fnode, Registry, :lookup, [rkey, pid], 15_000) + end end diff --git a/lib/supavisor/handlers/proxy/db.ex b/lib/supavisor/handlers/proxy/db.ex index c9d35c81..ce6b604c 100644 --- a/lib/supavisor/handlers/proxy/db.ex +++ b/lib/supavisor/handlers/proxy/db.ex @@ -3,10 +3,13 @@ defmodule Supavisor.Handlers.Proxy.Db do require Logger - alias Supavisor, as: S - alias Supavisor.Helpers, as: H - alias Supavisor.HandlerHelpers, as: HH - alias Supavisor.{Monitoring.Telem, Protocol.Server} + alias Supavisor.{ + Helpers, + DbHandler, + HandlerHelpers, + Monitoring.Telem, + Protocol.Server + } @type state :: :connect | :authentication | :idle | :busy @@ -16,7 +19,7 @@ defmodule Supavisor.Handlers.Proxy.Db do def handle_event(:info, {proto, _, bin}, :db_authentication, data) when proto in @proto do dec_pkt = Server.decode(bin) Logger.debug("ProxyDb: dec_pkt, #{inspect(dec_pkt, pretty: true)}") - HH.active_once(data.db_sock) + HandlerHelpers.active_once(data.db_sock) resp = Enum.reduce(dec_pkt, %{}, &handle_auth_pkts(&1, &2, data)) @@ -27,6 +30,15 @@ defmodule Supavisor.Handlers.Proxy.Db do {:authentication_server_first_message, server_proof} -> {:keep_state, %{data | server_proof: server_proof}} + %{authentication_server_final_message: _server_final} -> + :keep_state_and_data + + %{authentication_ok: true} -> + :keep_state_and_data + + :authentication -> + :keep_state_and_data + :authentication_md5 -> {:keep_state, data} @@ -56,15 +68,26 @@ defmodule Supavisor.Handlers.Proxy.Db do # forwards the message to the client def handle_event(:info, {proto, _, bin}, _, data) when proto in @proto do - HH.sock_send(data.sock, bin) - HH.active_once(data.db_sock) + HandlerHelpers.sock_send(data.sock, bin) + HandlerHelpers.active_once(data.db_sock) data = if String.ends_with?(bin, Server.ready_for_query()) do Logger.debug("ProxyDb: collected network usage") {_, stats} = Telem.network_usage(:client, data.sock, data.id, data.stats) {_, db_stats} = Telem.network_usage(:db, data.db_sock, data.id, data.db_stats) - %{data | stats: stats, db_stats: db_stats} + + case data.mode do + :transaction -> + DbHandler.set_idle(data.db_pid) + Helpers.controlling_process(data.db_sock, data.db_pid) + Process.unlink(data.db_pid) + :poolboy.checkin(data.pool, data.db_pid) + %{data | stats: stats, db_stats: db_stats, db_pid: nil, db_sock: nil} + + _ -> + %{data | stats: stats, db_stats: db_stats} + end else data end @@ -75,8 +98,13 @@ defmodule Supavisor.Handlers.Proxy.Db do def handle_event(_, {closed, _}, state, data) when closed in @sock_closed do Logger.error("ProxyDb: Connection closed when state was #{state}") Telem.handler_action(:db_handler, :stopped, data.id) - HH.sock_send(data.sock, Server.error_message("XX000", "Database connection closed")) - HH.sock_close(data.sock) + + HandlerHelpers.sock_send( + data.sock, + Server.error_message("XX000", "Database connection closed") + ) + + HandlerHelpers.sock_close(data.sock) {:stop, :db_socket_closed, data} end @@ -110,7 +138,7 @@ defmodule Supavisor.Handlers.Proxy.Db do ] bin = :pgo_protocol.encode_scram_response_message(sasl_initial_response) - :ok = HH.sock_send(data.db_sock, bin) + :ok = HandlerHelpers.sock_send(data.db_sock, bin) nonce other -> @@ -128,10 +156,10 @@ defmodule Supavisor.Handlers.Proxy.Db do ) when data.auth.require_user == false do nonce = data.nonce - server_first_parts = H.parse_server_first(server_first, nonce) + server_first_parts = Helpers.parse_server_first(server_first, nonce) {client_final_message, server_proof} = - H.get_client_final( + Helpers.get_client_final( :auth_query, data.auth.secrets.(), server_first_parts, @@ -141,7 +169,7 @@ defmodule Supavisor.Handlers.Proxy.Db do ) bin = :pgo_protocol.encode_scram_response_message(client_final_message) - :ok = HH.sock_send(data.db_sock, bin) + :ok = HandlerHelpers.sock_send(data.db_sock, bin) {:authentication_server_first_message, server_proof} end @@ -159,35 +187,42 @@ defmodule Supavisor.Handlers.Proxy.Db do server_first_parts, nonce, data.auth.user, - data.auth.password.() + data.auth.secrets.().password ) bin = :pgo_protocol.encode_scram_response_message(client_final_message) - :ok = HH.sock_send(data.db_sock, bin) + :ok = HandlerHelpers.sock_send(data.db_sock, bin) {:authentication_server_first_message, server_proof} end defp handle_auth_pkts( - %{payload: {:authentication_server_final_message, _server_final}}, + %{payload: {:authentication_server_final_message, server_final}}, + acc, + _data + ), + do: Map.put(acc, :authentication_server_final_message, server_final) + + defp handle_auth_pkts( + %{payload: :authentication_ok}, acc, _data ), - do: acc + do: Map.put(acc, :authentication_ok, true) defp handle_auth_pkts(%{payload: {:authentication_md5_password, salt}} = dec_pkt, _, data) do Logger.debug("ProxyDb: dec_pkt, #{inspect(dec_pkt, pretty: true)}") digest = if data.auth.method == :password do - H.md5([data.auth.password.(), data.auth.user]) + Helpers.md5([data.auth.password.(), data.auth.user]) else data.auth.secrets.().secret end - payload = ["md5", H.md5([digest, salt]), 0] + payload = ["md5", Helpers.md5([digest, salt]), 0] bin = [?p, <>, payload] - :ok = HH.sock_send(data.db_sock, bin) + :ok = HandlerHelpers.sock_send(data.db_sock, bin) :authentication_md5 end @@ -196,16 +231,17 @@ defmodule Supavisor.Handlers.Proxy.Db do defp handle_auth_pkts(_e, acc, _data), do: acc - @spec try_ssl_handshake(S.tcp_sock(), map()) :: {:ok, S.sock()} | {:error, term()} + @spec try_ssl_handshake(Supavisor.tcp_sock(), map()) :: + {:ok, Supavisor.sock()} | {:error, term()} def try_ssl_handshake(sock, %{upstream_ssl: true} = auth) do - with :ok <- HH.sock_send(sock, Server.ssl_request()) do + with :ok <- HandlerHelpers.sock_send(sock, Server.ssl_request()) do ssl_recv(sock, auth) end end def try_ssl_handshake(sock, _), do: {:ok, sock} - @spec ssl_recv(S.tcp_sock(), map) :: {:ok, S.ssl_sock()} | {:error, term} + @spec ssl_recv(Supavisor.tcp_sock(), map) :: {:ok, Supavisor.ssl_sock()} | {:error, term} def ssl_recv({:gen_tcp, sock} = s, auth) do case :gen_tcp.recv(sock, 1, 15_000) do {:ok, <>} -> ssl_connect(s, auth) @@ -214,7 +250,8 @@ defmodule Supavisor.Handlers.Proxy.Db do end end - @spec ssl_connect(S.tcp_sock(), map, pos_integer) :: {:ok, S.ssl_sock()} | {:error, term} + @spec ssl_connect(Supavisor.tcp_sock(), map, pos_integer) :: + {:ok, Supavisor.ssl_sock()} | {:error, term} def ssl_connect({:gen_tcp, sock}, auth, timeout \\ 5000) do opts = case auth.upstream_verify do @@ -237,9 +274,10 @@ defmodule Supavisor.Handlers.Proxy.Db do end end - @spec send_startup(S.sock(), map()) :: :ok | {:error, term} - def send_startup(sock, auth) do - user = get_user(auth) + @spec send_startup(Supavisor.sock(), map(), String.t() | nil) :: :ok | {:error, term} + def send_startup(sock, auth, tenant) do + user = + if is_nil(tenant), do: get_user(auth), else: "#{get_user(auth)}.#{tenant}" msg = :pgo_protocol.encode_startup_message([ @@ -248,7 +286,7 @@ defmodule Supavisor.Handlers.Proxy.Db do {"application_name", auth.application_name} ]) - HH.sock_send(sock, msg) + HandlerHelpers.sock_send(sock, msg) end @spec get_user(map) :: String.t() diff --git a/lib/supavisor/handlers/proxy/handler.ex b/lib/supavisor/handlers/proxy/handler.ex index 51d7601c..60b848d2 100644 --- a/lib/supavisor/handlers/proxy/handler.ex +++ b/lib/supavisor/handlers/proxy/handler.ex @@ -8,14 +8,13 @@ defmodule Supavisor.Handlers.Proxy.Handler do alias Supavisor.{ Helpers, + HandlerHelpers, Protocol.Server, Monitoring.PromEx, Handlers.Proxy.Db, Handlers.Proxy.Client } - alias Supavisor.HandlerHelpers, as: HH - @sock_closed [:tcp_closed, :ssl_closed] @proto [:tcp, :ssl] @@ -37,7 +36,7 @@ defmodule Supavisor.Handlers.Proxy.Handler do {:ok, sock} = :ranch.handshake(ref) :ok = trans.setopts(sock, active: true) - Logger.debug("ClientHandler is: #{inspect(self())}") + Logger.debug("ProxyHandler is: #{inspect(self())}") data = %{ id: nil, @@ -71,40 +70,39 @@ defmodule Supavisor.Handlers.Proxy.Handler do app_name: nil, peer_ip: Helpers.peer_ip(sock), auth: %{}, - backend_key_data: %{} + backend_key_data: %{}, + local: opts[:local] || false, + proxy: false } :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) end @impl true - def handle_event(e, {:client, _} = msg, state, data) do - Client.handle_event(e, msg, state, data) - end + def handle_event(:info, {:parameter_status, ps}, :exchange, _), + do: {:keep_state_and_data, {:next_event, :internal, {:client, {:greetings, ps}}}} - def handle_event(:timeout = e, msg, state, data) do - Client.handle_event(e, msg, state, data) - end + def handle_event(e, {:client, _} = msg, state, data), + do: Client.handle_event(e, msg, state, data) + + def handle_event(:timeout = e, msg, state, data), + do: Client.handle_event(e, msg, state, data) def handle_event(event, {proto, sock, _payload} = msg, state, %{sock: {_, sock}} = data) - when proto in @proto do - Client.handle_event(event, msg, state, data) - end + when proto in @proto, + do: Client.handle_event(event, msg, state, data) def handle_event(event, {proto, sock, _payload} = msg, state, %{db_sock: {_, sock}} = data) - when proto in @proto do - Db.handle_event(event, msg, state, data) - end + when proto in @proto, + do: Db.handle_event(event, msg, state, data) def handle_event(event, {closed, sock} = msg, state, %{sock: {_, sock}} = data) - when closed in @sock_closed do - Client.handle_event(event, msg, state, data) - end + when closed in @sock_closed, + do: Client.handle_event(event, msg, state, data) def handle_event(event, {closed, sock} = msg, state, %{db_sock: {_, sock}} = data) - when closed in @sock_closed do - Db.handle_event(event, msg, state, data) - end + when closed in @sock_closed, + do: Db.handle_event(event, msg, state, data) def handle_event(type, content, state, data) do msg = [ @@ -121,7 +119,7 @@ defmodule Supavisor.Handlers.Proxy.Handler do @impl true def terminate({:shutdown, reason}, state, data) do - HH.sock_send(data.sock, Server.error_message("XX000", "#{inspect(reason)}")) + HandlerHelpers.sock_send(data.sock, Server.error_message("XX000", "#{inspect(reason)}")) clean_up(data) Logger.info( @@ -143,12 +141,14 @@ defmodule Supavisor.Handlers.Proxy.Handler do @spec clean_up(map()) :: any() defp clean_up(data) do - HH.sock_close(data.sock) - HH.sock_close(data.db_sock) - - case Registry.lookup(Supavisor.Registry.TenantClients, data.id) do - clients when clients == [{self(), []}] or clients == [] -> PromEx.remove_metrics(data.id) - _ -> :ok + HandlerHelpers.sock_close(data.sock) + HandlerHelpers.sock_close(data.db_sock) + + if data.id != nil do + case Registry.lookup(Supavisor.Registry.TenantClients, data.id) do + clients when clients == [{self(), []}] or clients == [] -> PromEx.remove_metrics(data.id) + _ -> :ok + end end end end diff --git a/lib/supavisor/helpers.ex b/lib/supavisor/helpers.ex index b2bd9680..d0c8062f 100644 --- a/lib/supavisor/helpers.ex +++ b/lib/supavisor/helpers.ex @@ -350,14 +350,14 @@ defmodule Supavisor.Helpers do Process.flag(:max_heap_size, %{size: max_heap_words}) end - @spec set_log_level(atom()) :: :ok - def set_log_level(nil), do: :ok - - def set_log_level(level) when is_atom(level) do + @spec set_log_level(atom()) :: :ok | nil + def set_log_level(level) when level in [:debug, :info, :notice, :warning, :error] do Logger.notice("Setting log level to #{inspect(level)}") Logger.put_process_level(self(), level) end + def set_log_level(_), do: nil + @spec peer_ip(:gen_tcp.socket()) :: String.t() def peer_ip(socket) do case :inet.peername(socket) do @@ -365,4 +365,8 @@ defmodule Supavisor.Helpers do _error -> "undefined" end end + + @spec controlling_process(Supavisor.sock(), pid) :: :ok | {:error, any()} + def controlling_process({mod, socket}, pid), + do: mod.controlling_process(socket, pid) end diff --git a/lib/supavisor/monitoring/tenant.ex b/lib/supavisor/monitoring/tenant.ex index 55c12799..a4106cdc 100644 --- a/lib/supavisor/monitoring/tenant.ex +++ b/lib/supavisor/monitoring/tenant.ex @@ -27,6 +27,7 @@ defmodule Supavisor.PromEx.Plugins.Tenant do end defmodule Buckets do + @moduledoc false use Peep.Buckets.Custom, buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000] end diff --git a/lib/supavisor/pg_parser.ex b/lib/supavisor/pg_parser.ex index 961482a9..87640090 100644 --- a/lib/supavisor/pg_parser.ex +++ b/lib/supavisor/pg_parser.ex @@ -1,4 +1,6 @@ defmodule Supavisor.PgParser do + @moduledoc false + use Rustler, otp_app: :supavisor, crate: "pgparser" # When your NIF is loaded, it will override this function. diff --git a/lib/supavisor/protocol/client.ex b/lib/supavisor/protocol/client.ex index dc78e08f..307e114e 100644 --- a/lib/supavisor/protocol/client.ex +++ b/lib/supavisor/protocol/client.ex @@ -1,9 +1,12 @@ defmodule Supavisor.Protocol.Client do + @moduledoc false + require Logger @pkt_header_size 5 defmodule Pkt do + @moduledoc false defstruct([:tag, :len, :payload, :bin]) @type t :: %Pkt{ diff --git a/lib/supavisor/syn_handler.ex b/lib/supavisor/syn_handler.ex index 9ab5cf75..2f18bb62 100644 --- a/lib/supavisor/syn_handler.ex +++ b/lib/supavisor/syn_handler.ex @@ -9,11 +9,28 @@ defmodule Supavisor.SynHandler do :tenants, {{_type, _tenant}, _user, _mode, _db_name} = id, _pid, - _meta, + meta, reason ) do Logger.debug("Process unregistered: #{inspect(id)} #{inspect(reason)}") + case meta do + %{port: port, listener: listener} -> + try do + :ranch.stop_listener(id) + + Logger.notice( + "Stopped listener #{inspect(id)} on port #{inspect(port)} listener #{inspect(listener)}" + ) + rescue + exception -> + Logger.error("Failed to stop listener #{inspect(id)} #{Exception.message(exception)}") + end + + _ -> + nil + end + # remove all Prometheus metrics for the specified tenant PromEx.remove_metrics(id) end diff --git a/lib/supavisor/tenant_supervisor.ex b/lib/supavisor/tenant_supervisor.ex index c3b8c3a9..21d518ce 100644 --- a/lib/supavisor/tenant_supervisor.ex +++ b/lib/supavisor/tenant_supervisor.ex @@ -2,8 +2,16 @@ defmodule Supavisor.TenantSupervisor do @moduledoc false use Supervisor + require Logger alias Supavisor.Manager + def start_link(%{replicas: [%{mode: :transaction} = single]} = args) do + {:ok, meta} = Supavisor.start_local_server(single) + Logger.info("Starting ranch instance #{inspect(meta)} for #{inspect(args.id)}") + name = {:via, :syn, {:tenants, args.id, meta}} + Supervisor.start_link(__MODULE__, args, name: name) + end + def start_link(args) do name = {:via, :syn, {:tenants, args.id}} Supervisor.start_link(__MODULE__, args, name: name) @@ -57,6 +65,7 @@ defmodule Supavisor.TenantSupervisor do # end {size, overflow} = {1, args.pool_size} + # {size, overflow} = {args.pool_size, 0} [ name: {:via, Registry, {Supavisor.Registry.Tenants, id, args.replica_type}}, diff --git a/lib/supavisor/tenants/cluster.ex b/lib/supavisor/tenants/cluster.ex index eb7e5de6..b4ad69f2 100644 --- a/lib/supavisor/tenants/cluster.ex +++ b/lib/supavisor/tenants/cluster.ex @@ -1,4 +1,6 @@ defmodule Supavisor.Tenants.Cluster do + @moduledoc false + use Ecto.Schema import Ecto.Changeset alias Supavisor.Tenants.ClusterTenants diff --git a/lib/supavisor/tenants/cluster_tenants.ex b/lib/supavisor/tenants/cluster_tenants.ex index 10c57759..0b247aa1 100644 --- a/lib/supavisor/tenants/cluster_tenants.ex +++ b/lib/supavisor/tenants/cluster_tenants.ex @@ -1,4 +1,6 @@ defmodule Supavisor.Tenants.ClusterTenants do + @moduledoc false + use Ecto.Schema import Ecto.Changeset alias Supavisor.Tenants.Tenant diff --git a/mix.lock b/mix.lock index 69db459e..551946e2 100644 --- a/mix.lock +++ b/mix.lock @@ -41,7 +41,7 @@ "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "observer_cli": {:hex, :observer_cli, "1.7.4", "3c1bfb6d91bf68f6a3d15f46ae20da0f7740d363ee5bc041191ce8722a6c4fae", [:mix, :rebar3], [{:recon, "~> 2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "50de6d95d814f447458bd5d72666a74624eddb0ef98bdcee61a0153aae0865ff"}, "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"}, - "open_api_spex": {:hex, :open_api_spex, "3.19.1", "65ccb5d06e3d664d1eec7c5ea2af2289bd2f37897094a74d7219fb03fc2b5994", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "392895827ce2984a3459c91a484e70708132d8c2c6c5363972b4b91d6bbac3dd"}, + "open_api_spex": {:hex, :open_api_spex, "3.20.0", "d4fcf1ee297aa94a673cddb92734eb0bc7cac698be93949a223a50f724e3af89", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "2e9beea71142ff09f8f935579b39406e2c6b5a3978e7235978d7faf2f90cd081"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.3.0", "03e2177f28dd8d11aaa88e8522c81c2f6a788170fe52f7a65262340961e663f9", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "b9e5ff775fd064fa098dba3c398490b77649a352b40b0b730a6b7dc0bdd68858"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, "peep": {:hex, :peep, "3.1.0", "1680337d682dfde308b643814834379a5210eb11db5aaca8ea823c218d4d7e16", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "3de3c13f3efcff6130e53c8956d41c20a436650cfddee6e8aa6ebdef751d542f"}, @@ -64,7 +64,7 @@ "prom_ex": {:git, "https://github.com/hauleth/prom_ex.git", "d6b41ffc6ba77a1dbbbf601eaa76af15a066c101", [branch: "ft/add-peep-storage"]}, "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, "recon": {:hex, :recon, "2.5.5", "c108a4c406fa301a529151a3bb53158cadc4064ec0c5f99b03ddb8c0e4281bdf", [:mix, :rebar3], [], "hexpm", "632a6f447df7ccc1a4a10bdcfce71514412b16660fe59deca0fcf0aa3c054404"}, - "req": {:hex, :req, "0.5.2", "70b4976e5fbefe84e5a57fd3eea49d4e9aa0ac015301275490eafeaec380f97f", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0c63539ab4c2d6ced6114d2684276cef18ac185ee00674ee9af4b1febba1f986"}, + "req": {:hex, :req, "0.5.4", "e375e4812adf83ffcf787871d7a124d873e983e3b77466e6608b973582f7f837", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "a17998ffe2ef54f79bfdd782ef9f4cbf987d93851e89444cbc466a6a25eee494"}, "rustler": {:hex, :rustler, "0.34.0", "e9a73ee419fc296a10e49b415a2eb87a88c9217aa0275ec9f383d37eed290c1c", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "1d0c7449482b459513003230c0e2422b0252245776fe6fd6e41cb2b11bd8e628"}, "sleeplocks": {:hex, :sleeplocks, "1.1.3", "96a86460cc33b435c7310dbd27ec82ca2c1f24ae38e34f8edde97f756503441a", [:rebar3], [], "hexpm", "d3b3958552e6eb16f463921e70ae7c767519ef8f5be46d7696cc1ed649421321"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, diff --git a/priv/repo/seeds_after_migration.exs b/priv/repo/seeds_after_migration.exs index 3a99bfe1..51873ecd 100644 --- a/priv/repo/seeds_after_migration.exs +++ b/priv/repo/seeds_after_migration.exs @@ -54,6 +54,7 @@ end "db_user" => db_conf[:username], "db_password" => db_conf[:password], "pool_size" => 9, + "max_clients" => 100, "mode_type" => "transaction" }, %{ @@ -61,6 +62,7 @@ end "db_user" => db_conf[:username], "db_password" => db_conf[:password], "pool_size" => 3, + "max_clients" => 100, "mode_type" => "transaction" }, %{ @@ -69,6 +71,7 @@ end "db_password" => db_conf[:password], "pool_size" => 1, "mode_type" => "session", + "max_clients" => 100, "pool_checkout_timeout" => 500 }, %{ diff --git a/test/integration/proxy_test.exs b/test/integration/proxy_test.exs index aaee5ff7..b3031490 100644 --- a/test/integration/proxy_test.exs +++ b/test/integration/proxy_test.exs @@ -4,6 +4,7 @@ defmodule Supavisor.Integration.ProxyTest do require Logger alias Postgrex, as: P + alias Supavisor.Support.Cluster @tenant "proxy_tenant1" @@ -75,7 +76,7 @@ defmodule Supavisor.Integration.ProxyTest do end test "query via another node", %{proxy: proxy, user: user} do - {:ok, _pid, node2} = Supavisor.Support.Cluster.start_node() + {:ok, _pid, node2} = Cluster.start_node() sup = Enum.reduce_while(1..30, nil, fn _, acc -> @@ -265,8 +266,7 @@ defmodule Supavisor.Integration.ProxyTest do %Postgrex.Error{ postgres: %{ code: :internal_error, - message: - "Authentication error, reason: \"Invalid characters in user or db_name\"", + message: "Authentication error, reason: \"Invalid format for user or db_name\"", pg_code: "XX000", severity: "FATAL", unknown: "FATAL" diff --git a/test/supavisor/syn_handler_test.exs b/test/supavisor/syn_handler_test.exs index 2404e2c1..90d40657 100644 --- a/test/supavisor/syn_handler_test.exs +++ b/test/supavisor/syn_handler_test.exs @@ -3,11 +3,12 @@ defmodule Supavisor.SynHandlerTest do import ExUnit.CaptureLog require Logger alias Ecto.Adapters.SQL.Sandbox + alias Supavisor.Support.Cluster @id {{:single, "syn_tenant"}, "postgres", :session, "postgres"} test "resolving conflict" do - {:ok, _pid, node2} = Supavisor.Support.Cluster.start_node() + {:ok, _pid, node2} = Cluster.start_node() secret = %{alias: "postgres"} auth_secret = {:password, fn -> secret end} diff --git a/test/supavisor_web/controllers/metrics_controller_test.exs b/test/supavisor_web/controllers/metrics_controller_test.exs index 9bb0963f..06f7a978 100644 --- a/test/supavisor_web/controllers/metrics_controller_test.exs +++ b/test/supavisor_web/controllers/metrics_controller_test.exs @@ -1,5 +1,6 @@ defmodule SupavisorWeb.MetricsControllerTest do use SupavisorWeb.ConnCase + alias Supavisor.Support.Cluster setup %{conn: conn} do new_conn = @@ -13,7 +14,7 @@ defmodule SupavisorWeb.MetricsControllerTest do end test "exporting metrics", %{conn: conn} do - {:ok, _pid, node2} = Supavisor.Support.Cluster.start_node() + {:ok, _pid, node2} = Cluster.start_node() Node.connect(node2) diff --git a/test/support/fixtures/single_connection.ex b/test/support/fixtures/single_connection.ex index 1b416c50..887717ca 100644 --- a/test/support/fixtures/single_connection.ex +++ b/test/support/fixtures/single_connection.ex @@ -1,4 +1,6 @@ defmodule SingleConnection do + @moduledoc false + alias Postgrex, as: P @behaviour P.SimpleConnection From 6e54020d307caee55f16d4ef833ed03fa9aa3cc9 Mon Sep 17 00:00:00 2001 From: Stas Date: Thu, 1 Aug 2024 14:02:13 +0200 Subject: [PATCH 30/97] chore: get rid of short names (#409) --- config/runtime.exs | 4 +- lib/supavisor.ex | 31 ++-- lib/supavisor/client_handler.ex | 140 ++++++++++-------- lib/supavisor/db_handler.ex | 45 +++--- lib/supavisor/handler_helpers.ex | 31 ++-- .../controllers/tenant_controller.ex | 9 +- 6 files changed, 132 insertions(+), 128 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index cf0bf381..e3ef8dab 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1,7 +1,7 @@ import Config require Logger -alias Supavisor.Helpers, as: H +alias Supavisor.Helpers secret_key_base = if config_env() in [:dev, :test] do @@ -101,7 +101,7 @@ config :libcluster, upstream_ca = if path = System.get_env("GLOBAL_UPSTREAM_CA_PATH") do File.read!(path) - |> H.cert_to_bin() + |> Helpers.cert_to_bin() |> case do {:ok, bin} -> Logger.info("Loaded upstream CA from $GLOBAL_UPSTREAM_CA_PATH", diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 76fb517e..8324552c 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -2,9 +2,7 @@ defmodule Supavisor do @moduledoc false require Logger import Cachex.Spec - alias Supavisor.Helpers, as: H - alias Supavisor.Tenants, as: T - alias Supavisor.Manager + alias Supavisor.{Manager, Helpers, Tenants} @type sock :: tcp_sock() | ssl_sock() @type ssl_sock :: {:ssl, :ssl.sslsocket()} @@ -32,7 +30,7 @@ defmodule Supavisor do start_local_pool(id, secrets, log_level) else Logger.debug("Starting remote pool for #{inspect(id)}") - H.rpc(node, __MODULE__, :start_local_pool, [id, secrets, log_level]) + Helpers.rpc(node, __MODULE__, :start_local_pool, [id, secrets, log_level]) end pid -> @@ -89,7 +87,7 @@ defmodule Supavisor do if node() == dest_node do subscribe_local(pid, id) else - H.rpc(dest_node, __MODULE__, :subscribe_local, [pid, id], 15_000) + Helpers.rpc(dest_node, __MODULE__, :subscribe_local, [pid, id], 15_000) end end @@ -241,18 +239,18 @@ defmodule Supavisor do user = elem(secrets, 1).().alias case type do - :single -> T.get_pool_config(tenant, user) - :cluster -> T.get_cluster_config(tenant, user) + :single -> Tenants.get_pool_config(tenant, user) + :cluster -> Tenants.get_cluster_config(tenant, user) end |> case do [_ | _] = replicas -> opts = Enum.map(replicas, fn replica -> case replica do - %T.ClusterTenants{tenant: tenant, type: type} -> + %Tenants.ClusterTenants{tenant: tenant, type: type} -> Map.put(tenant, :replica_type, type) - %T.Tenant{} = tenant -> + %Tenants.Tenant{} = tenant -> Map.put(tenant, :replica_type, :write) end |> supervisor_args(id, secrets, log_level) @@ -317,10 +315,10 @@ defmodule Supavisor do database: if(db_name != nil, do: db_name, else: db_database), password: fn -> db_pass end, application_name: "Supavisor", - ip_version: H.ip_version(ip_ver, db_host), + ip_version: Helpers.ip_version(ip_ver, db_host), upstream_ssl: tenant_record.upstream_ssl, upstream_verify: tenant_record.upstream_verify, - upstream_tls_ca: H.upstream_cert(tenant_record.upstream_tls_ca), + upstream_tls_ca: Helpers.upstream_cert(tenant_record.upstream_tls_ca), require_user: tenant_record.require_user, method: method, secrets: secrets @@ -366,12 +364,11 @@ defmodule Supavisor do do: {ceil(max_clients / 100), max_clients}, else: {1, 100} - opts = - %{ - max_connections: max_clients, - num_acceptors: max(acceptors, 10), - socket_opts: [port: 0, keepalive: true] - } + opts = %{ + max_connections: max_clients, + num_acceptors: max(acceptors, 10), + socket_opts: [port: 0, keepalive: true] + } handler = Supavisor.Handlers.Proxy.Handler args = Map.put(args, :local, true) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index aa8b270d..90b85ab1 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -11,11 +11,15 @@ defmodule Supavisor.ClientHandler do @behaviour :ranch_protocol @behaviour :gen_statem - alias Supavisor, as: S - alias Supavisor.DbHandler, as: Db - alias Supavisor.Helpers, as: H - alias Supavisor.HandlerHelpers, as: HH - alias Supavisor.{Tenants, Monitoring.Telem, Protocol.Client, Protocol.Server} + alias Supavisor.{ + Tenants, + Helpers, + DbHandler, + HandlerHelpers, + Protocol.Client, + Protocol.Server, + Monitoring.Telem + } @impl true def start_link(ref, transport, opts) do @@ -39,7 +43,7 @@ defmodule Supavisor.ClientHandler do def init(ref, trans, opts) do Process.flag(:trap_exit, true) - H.set_max_heap_size(150) + Helpers.set_max_heap_size(150) {:ok, sock} = :ranch.handshake(ref) :ok = trans.setopts(sock, active: true) @@ -77,7 +81,7 @@ defmodule Supavisor.ClientHandler do def handle_event(:info, {_proto, _, <<"GET", _::binary>>}, :exchange, data) do Logger.debug("ClientHandler: Client is trying to request HTTP") - HH.sock_send( + HandlerHelpers.sock_send( data.sock, "HTTP/1.1 204 OK\r\nx-app-version: #{Application.spec(:supavisor, :vsn)}\r\n\r\n" ) @@ -88,7 +92,7 @@ defmodule Supavisor.ClientHandler do # cancel request def handle_event(:info, {_, _, <<16::32, 1234::16, 5678::16, pid::32, key::32>>}, _, _) do Logger.debug("ClientHandler: Got cancel query for #{inspect({pid, key})}") - :ok = HH.send_cancel_query(pid, key) + :ok = HandlerHelpers.send_cancel_query(pid, key) {:stop, {:shutdown, :cancel_query}} end @@ -100,7 +104,7 @@ defmodule Supavisor.ClientHandler do case db_pid_meta(key) do [{^db_pid, meta}] -> - :ok = HH.cancel_query(meta.host, meta.port, meta.ip_ver, meta.pid, meta.key) + :ok = HandlerHelpers.cancel_query(meta.host, meta.port, meta.ip_ver, meta.pid, meta.key) error -> Logger.error( @@ -114,13 +118,13 @@ defmodule Supavisor.ClientHandler do def handle_event(:info, {:tcp, _, <<_::64>>}, :exchange, %{sock: sock} = data) do Logger.debug("ClientHandler: Client is trying to connect with SSL") - downstream_cert = H.downstream_cert() - downstream_key = H.downstream_key() + downstream_cert = Helpers.downstream_cert() + downstream_key = Helpers.downstream_key() # SSL negotiation, S/N/Error if !!downstream_cert and !!downstream_key do - :ok = HH.setopts(sock, active: false) - :ok = HH.sock_send(sock, "S") + :ok = HandlerHelpers.setopts(sock, active: false) + :ok = HandlerHelpers.sock_send(sock, "S") opts = [ certfile: downstream_cert, @@ -130,7 +134,7 @@ defmodule Supavisor.ClientHandler do case :ssl.handshake(elem(sock, 1), opts) do {:ok, ssl_sock} -> socket = {:ssl, ssl_sock} - :ok = HH.setopts(socket, active: true) + :ok = HandlerHelpers.setopts(socket, active: true) {:keep_state, %{data | sock: socket, ssl: true}} error -> @@ -143,7 +147,7 @@ defmodule Supavisor.ClientHandler do "ClientHandler: User requested SSL connection but no downstream cert/key found" ) - :ok = HH.sock_send(data.sock, "N") + :ok = HandlerHelpers.sock_send(data.sock, "N") :keep_state_and_data end end @@ -152,7 +156,7 @@ defmodule Supavisor.ClientHandler do case Server.decode_startup_packet(bin) do {:ok, hello} -> Logger.debug("ClientHandler: Client startup message: #{inspect(hello)}") - {type, {user, tenant_or_alias, db_name}} = HH.parse_user_info(hello.payload) + {type, {user, tenant_or_alias, db_name}} = HandlerHelpers.parse_user_info(hello.payload) not_allowed = ["\"", "\\"] @@ -160,7 +164,13 @@ defmodule Supavisor.ClientHandler do reason = "Invalid characters in user or db_name" Logger.error("ClientHandler: #{inspect(reason)}") Telem.client_join(:fail, data.id) - HH.send_error(data.sock, "XX000", "Authentication error, reason: #{inspect(reason)}") + + HandlerHelpers.send_error( + data.sock, + "XX000", + "Authentication error, reason: #{inspect(reason)}" + ) + {:stop, {:shutdown, :invalid_characters}} else log_level = @@ -169,7 +179,7 @@ defmodule Supavisor.ClientHandler do level -> String.to_existing_atom(level) end - H.set_log_level(log_level) + Helpers.set_log_level(log_level) {:keep_state, %{data | log_level: log_level}, {:next_event, :internal, {:hello, {type, {user, tenant_or_alias, db_name}}}}} @@ -188,7 +198,7 @@ defmodule Supavisor.ClientHandler do :exchange, %{sock: sock} = data ) do - sni_hostname = HH.try_get_sni(sock) + sni_hostname = HandlerHelpers.try_get_sni(sock) case Tenants.get_user_cache(type, user, tenant_or_alias, sni_hostname) do {:ok, info} -> @@ -203,7 +213,7 @@ defmodule Supavisor.ClientHandler do db_name ) - mode = S.mode(id) + mode = Supavisor.mode(id) Logger.metadata( project: tenant_or_alias, @@ -215,7 +225,7 @@ defmodule Supavisor.ClientHandler do Registry.register(Supavisor.Registry.TenantClients, id, []) - {:ok, addr} = HH.addr_from_sock(sock) + {:ok, addr} = HandlerHelpers.addr_from_sock(sock) cond do info.tenant.enforce_ssl and !data.ssl -> @@ -223,14 +233,14 @@ defmodule Supavisor.ClientHandler do "ClientHandler: Tenant is not allowed to connect without SSL, user #{user}" ) - :ok = HH.send_error(sock, "XX000", "SSL connection is required") + :ok = HandlerHelpers.send_error(sock, "XX000", "SSL connection is required") Telem.client_join(:fail, id) {:stop, {:shutdown, :ssl_required}} - HH.filter_cidrs(info.tenant.allow_list, addr) == [] -> + HandlerHelpers.filter_cidrs(info.tenant.allow_list, addr) == [] -> message = "Address not in tenant allow_list: " <> inspect(addr) Logger.error("ClientHandler: #{message}") - :ok = HH.send_error(sock, "XX000", message) + :ok = HandlerHelpers.send_error(sock, "XX000", message) Telem.client_join(:fail, id) {:stop, {:shutdown, :address_not_allowed}} @@ -252,7 +262,11 @@ defmodule Supavisor.ClientHandler do ) :ok = - HH.send_error(sock, "XX000", "Authentication error, reason: #{inspect(reason)}") + HandlerHelpers.send_error( + sock, + "XX000", + "Authentication error, reason: #{inspect(reason)}" + ) Telem.client_join(:fail, id) {:stop, {:shutdown, :auth_secrets_error}} @@ -264,7 +278,7 @@ defmodule Supavisor.ClientHandler do "ClientHandler: User not found: #{inspect(reason)} #{inspect({type, user, tenant_or_alias})}" ) - :ok = HH.send_error(sock, "XX000", "Tenant or user not found") + :ok = HandlerHelpers.send_error(sock, "XX000", "Tenant or user not found") Telem.client_join(:fail, data.id) {:stop, {:shutdown, :user_not_found}} end @@ -318,7 +332,7 @@ defmodule Supavisor.ClientHandler do Logger.debug("ClientHandler: Cache hit for #{inspect(key)}") end - HH.sock_send(sock, msg) + HandlerHelpers.sock_send(sock, msg) Telem.client_join(:fail, data.id) {:stop, {:shutdown, :exchange_error}} @@ -333,7 +347,7 @@ defmodule Supavisor.ClientHandler do end Logger.debug("ClientHandler: Exchange success") - :ok = HH.sock_send(sock, Server.authentication_ok()) + :ok = HandlerHelpers.sock_send(sock, Server.authentication_ok()) Telem.client_join(:ok, data.id) {:keep_state, %{data | auth_secrets: {method, secrets}}, @@ -364,7 +378,7 @@ defmodule Supavisor.ClientHandler do {:error, :max_clients_reached} -> msg = "Max client connections reached" Logger.error("ClientHandler: #{msg}") - :ok = HH.send_error(data.sock, "XX000", msg) + :ok = HandlerHelpers.send_error(data.sock, "XX000", msg) Telem.client_join(:fail, data.id) {:stop, {:shutdown, :max_clients_reached}} @@ -377,8 +391,8 @@ defmodule Supavisor.ClientHandler do def handle_event(:internal, {:greetings, ps}, _, %{sock: sock} = data) do {header, <> = payload} = Server.backend_key_data() msg = [ps, [header, payload], Server.ready_for_query()] - :ok = HH.listen_cancel_query(pid, key) - :ok = HH.sock_send(sock, msg) + :ok = HandlerHelpers.listen_cancel_query(pid, key) + :ok = HandlerHelpers.sock_send(sock, msg) Telem.client_connection_time(data.connection_start, data.id) {:next_state, :idle, data, handle_actions(data)} end @@ -403,7 +417,7 @@ defmodule Supavisor.ClientHandler do def handle_event(:timeout, :heartbeat_check, _, data) do Logger.debug("ClientHandler: Send heartbeat to client") - HH.sock_send(data.sock, Server.application_name()) + HandlerHelpers.sock_send(data.sock, Server.application_name()) {:keep_state_and_data, {:timeout, data.heartbeat_interval, :heartbeat_check}} end @@ -418,7 +432,7 @@ defmodule Supavisor.ClientHandler do def handle_event(:info, {proto, _, <>}, :idle, data) when proto in [:tcp, :ssl] do Logger.debug("ClientHandler: Receive sync") - :ok = HH.sock_send(data.sock, Server.ready_for_query()) + :ok = HandlerHelpers.sock_send(data.sock, Server.ready_for_query()) {:keep_state_and_data, handle_actions(data)} end @@ -426,7 +440,7 @@ defmodule Supavisor.ClientHandler do when proto in [:tcp, :ssl] do Logger.debug("ClientHandler: Receive sync while not idle") {_, db_pid} = data.db_pid - Db.cast(db_pid, self(), msg) + DbHandler.cast(db_pid, self(), msg) :keep_state_and_data end @@ -434,7 +448,7 @@ defmodule Supavisor.ClientHandler do when proto in [:tcp, :ssl] do Logger.debug("ClientHandler: Receive flush while not idle") {_, db_pid} = data.db_pid - Db.cast(db_pid, self(), msg) + DbHandler.cast(db_pid, self(), msg) :keep_state_and_data end @@ -481,7 +495,7 @@ defmodule Supavisor.ClientHandler do when proto in [:tcp, :ssl] do {_, db_pid} = data.db_pid - case Db.call(db_pid, self(), bin) do + case DbHandler.call(db_pid, self(), bin) do :ok -> Logger.debug("ClientHandler: DbHandler call success") :keep_state_and_data @@ -492,7 +506,7 @@ defmodule Supavisor.ClientHandler do if size > 1_000_000 do msg = "DbHandler buffer size is too big: #{size}" Logger.error("ClientHandler: #{msg}") - HH.sock_send(data.sock, Server.error_message("XX000", msg)) + HandlerHelpers.sock_send(data.sock, Server.error_message("XX000", msg)) {:stop, {:shutdown, :buffer_size}} else Logger.debug("ClientHandler: DbHandler call buffering") @@ -502,7 +516,7 @@ defmodule Supavisor.ClientHandler do {:error, reason} -> msg = "DbHandler error: #{inspect(reason)}" Logger.error("ClientHandler: #{msg}") - HH.sock_send(data.sock, Server.error_message("XX000", msg)) + HandlerHelpers.sock_send(data.sock, Server.error_message("XX000", msg)) {:stop, {:shutdown, :db_handler_error}} end end @@ -526,7 +540,7 @@ defmodule Supavisor.ClientHandler do # linked DbHandler went down def handle_event(:info, {:EXIT, db_pid, reason}, _, data) do Logger.error("ClientHandler: DbHandler #{inspect(db_pid)} exited #{inspect(reason)}") - HH.sock_send(data.sock, Server.error_message("XX000", "DbHandler exited")) + HandlerHelpers.sock_send(data.sock, Server.error_message("XX000", "DbHandler exited")) {:stop, {:shutdown, :db_handler_exit}} end @@ -566,13 +580,13 @@ defmodule Supavisor.ClientHandler do {_, stats} = Telem.network_usage(:client, data.sock, data.id, data.stats) Telem.client_query_time(data.query_start, data.id) - :ok = HH.sock_send(data.sock, bin) + :ok = HandlerHelpers.sock_send(data.sock, bin) actions = handle_actions(data) {:next_state, :idle, %{data | db_pid: db_pid, stats: stats}, actions} :continue -> Logger.debug("ClientHandler: Client is not ready") - :ok = HH.sock_send(data.sock, bin) + :ok = HandlerHelpers.sock_send(data.sock, bin) :keep_state_and_data :read_sql_error -> @@ -594,7 +608,7 @@ defmodule Supavisor.ClientHandler do # emulate handle_call def handle_event({:call, from}, {:client_call, bin, _}, _, data) do Logger.debug("ClientHandler: --> --> bin call #{inspect(byte_size(bin))} bytes") - {:keep_state_and_data, {:reply, from, HH.sock_send(data.sock, bin)}} + {:keep_state_and_data, {:reply, from, HandlerHelpers.sock_send(data.sock, bin)}} end def handle_event(type, content, state, data) do @@ -626,14 +640,14 @@ defmodule Supavisor.ClientHandler do end Logger.error("ClientHandler: #{msg}") - HH.sock_send(data.sock, Server.error_message("XX000", msg)) + HandlerHelpers.sock_send(data.sock, Server.error_message("XX000", msg)) :ok end def terminate(reason, _state, %{db_pid: {_, pid}}) do db_info = - with {:ok, {state, mode} = resp} <- Db.get_state_and_mode(pid) do - if state == :busy or mode == :session, do: Db.stop(pid) + with {:ok, {state, mode} = resp} <- DbHandler.get_state_and_mode(pid) do + if state == :busy or mode == :session, do: DbHandler.stop(pid) resp end @@ -651,10 +665,11 @@ defmodule Supavisor.ClientHandler do ## Internal functions - @spec handle_exchange(S.sock(), {atom(), fun()}) :: {:ok, binary() | nil} | {:error, String.t()} + @spec handle_exchange(Supavisor.sock(), {atom(), fun()}) :: + {:ok, binary() | nil} | {:error, String.t()} def handle_exchange({_, socket} = sock, {:auth_query_md5 = method, secrets}) do salt = :crypto.strong_rand_bytes(4) - :ok = HH.sock_send(sock, Server.md5_request(salt)) + :ok = HandlerHelpers.sock_send(sock, Server.md5_request(salt)) with {:ok, %{ @@ -670,7 +685,7 @@ defmodule Supavisor.ClientHandler do end def handle_exchange({_, socket} = sock, {method, secrets}) do - :ok = HH.sock_send(sock, Server.scram_request()) + :ok = HandlerHelpers.sock_send(sock, Server.scram_request()) with {:ok, %{ @@ -695,7 +710,7 @@ defmodule Supavisor.ClientHandler do ), {:ok, key} <- authenticate_exchange(method, secrets, signatures, p) do message = "v=#{Base.encode64(signatures.server)}" - :ok = HH.sock_send(sock, Server.exchange_message(:final, message)) + :ok = HandlerHelpers.sock_send(sock, Server.exchange_message(:final, message)) {:ok, key} else {:error, message} -> {:error, message} @@ -717,7 +732,7 @@ defmodule Supavisor.ClientHandler do defp reply_first_exchange(sock, method, secrets, channel, nonce, user) do {message, signatures} = exchange_first(method, secrets, nonce, user, channel) - :ok = HH.sock_send(sock, Server.exchange_message(:first, message)) + :ok = HandlerHelpers.sock_send(sock, Server.exchange_message(:first, message)) {:ok, signatures} end @@ -732,7 +747,7 @@ defmodule Supavisor.ClientHandler do defp authenticate_exchange(:auth_query, secrets, signatures, p) do client_key = :crypto.exor(Base.decode64!(p), signatures.client) - if H.hash(client_key) == secrets.().stored_key do + if Helpers.hash(client_key) == secrets.().stored_key do {:ok, client_key} else {:error, "Wrong password"} @@ -740,7 +755,7 @@ defmodule Supavisor.ClientHandler do end defp authenticate_exchange(:auth_query_md5, client_hash, server_hash, salt) do - if "md5" <> H.md5([server_hash, salt]) == client_hash do + if "md5" <> Helpers.md5([server_hash, salt]) == client_hash do {:ok, nil} else {:error, "Wrong password"} @@ -810,7 +825,7 @@ defmodule Supavisor.ClientHandler do end @spec auth_secrets(map, String.t(), term(), non_neg_integer()) :: - {:ok, S.secrets()} | {:error, term()} + {:ok, Supavisor.secrets()} | {:error, term()} ## password secrets def auth_secrets(%{user: user, tenant: %{require_user: true}}, _, _, _) do secrets = %{db_user: user.db_user, password: user.db_password, alias: user.db_user_alias} @@ -840,7 +855,7 @@ defmodule Supavisor.ClientHandler do if tenant.upstream_ssl and tenant.upstream_verify == "peer" do [ {:verify, :verify_peer}, - {:cacerts, [H.upstream_cert(tenant.upstream_tls_ca)]}, + {:cacerts, [Helpers.upstream_cert(tenant.upstream_tls_ca)]}, {:server_name_indication, String.to_charlist(tenant.db_host)}, {:customize_hostname_check, [{:match_fun, fn _, _ -> true end}]} ] @@ -856,7 +871,7 @@ defmodule Supavisor.ClientHandler do parameters: [application_name: "Supavisor auth_query"], ssl: tenant.upstream_ssl, socket_options: [ - H.ip_version(tenant.ip_version, tenant.db_host) + Helpers.ip_version(tenant.ip_version, tenant.db_host) ], queue_target: 1_000, queue_interval: 5_000, @@ -864,7 +879,7 @@ defmodule Supavisor.ClientHandler do ) resp = - case H.get_user_secret(conn, tenant.auth_query, db_user) do + case Helpers.get_user_secret(conn, tenant.auth_query, db_user) do {:ok, secret} -> t = if secret.digest == :md5, do: :auth_query_md5, else: :auth_query {:ok, {t, fn -> Map.put(secret, :alias, user.db_user_alias) end}} @@ -881,10 +896,10 @@ defmodule Supavisor.ClientHandler do {binary(), map()} defp exchange_first(:password, secret, nonce, user, channel) do message = Server.exchange_first_message(nonce) - server_first_parts = H.parse_server_first(message, nonce) + server_first_parts = Helpers.parse_server_first(message, nonce) {client_final_message, server_proof} = - H.get_client_final( + Helpers.get_client_final( :password, secret.().password, server_first_parts, @@ -904,10 +919,10 @@ defmodule Supavisor.ClientHandler do defp exchange_first(:auth_query, secret, nonce, user, channel) do secret = secret.() message = Server.exchange_first_message(nonce, secret.salt) - server_first_parts = H.parse_server_first(message, nonce) + server_first_parts = Helpers.parse_server_first(message, nonce) sings = - H.signatures( + Helpers.signatures( secret.stored_key, secret.server_key, server_first_parts, @@ -919,7 +934,7 @@ defmodule Supavisor.ClientHandler do {message, sings} end - @spec try_get_sni(S.sock()) :: String.t() | nil + @spec try_get_sni(Supavisor.sock()) :: String.t() | nil def try_get_sni({:ssl, sock}) do case :ssl.connection_information(sock, [:sni_hostname]) do {:ok, [sni_hostname: sni]} -> List.to_string(sni) @@ -975,8 +990,7 @@ defmodule Supavisor.ClientHandler do do: [{:timeout, data.heartbeat_interval, :heartbeat_check}], else: [] - idle = - if data.idle_timeout > 0, do: [{:timeout, data.idle_timeout, :idle_timeout}], else: [] + idle = if data.idle_timeout > 0, do: [{:timeout, data.idle_timeout, :idle_timeout}], else: [] idle ++ heartbeat end diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 2af17bae..9c36dce6 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -8,11 +8,7 @@ defmodule Supavisor.DbHandler do @behaviour :gen_statem - alias Supavisor, as: S - alias Supavisor.ClientHandler, as: Client - alias Supavisor.Helpers, as: H - alias Supavisor.HandlerHelpers, as: HH - alias Supavisor.{Monitoring.Telem, Protocol.Server} + alias Supavisor.{Helpers, HandlerHelpers, ClientHandler, Monitoring.Telem, Protocol.Server} @type state :: :connect | :authentication | :idle | :busy @@ -34,7 +30,7 @@ defmodule Supavisor.DbHandler do @spec set_idle(pid()) :: :ok def set_idle(pid), do: :gen_statem.cast(pid, :set_idle) - @spec change_socket_owner(pid(), pid()) :: {:ok, S.sock()} + @spec change_socket_owner(pid(), pid()) :: {:ok, Supavisor.sock()} def change_socket_owner(pid, caller), do: :gen_statem.call(pid, {:tcp_owner, caller}, 15_000) @spec get_state_and_mode(pid()) :: {:ok, {state, Supavisor.mode()}} | {:error, term()} @@ -52,8 +48,8 @@ defmodule Supavisor.DbHandler do @impl true def init(args) do Process.flag(:trap_exit, true) - H.set_log_level(args.log_level) - H.set_max_heap_size(150) + Helpers.set_log_level(args.log_level) + Helpers.set_max_heap_size(150) {_, tenant} = args.tenant Logger.metadata(project: tenant, user: args.user, mode: args.mode) @@ -187,10 +183,10 @@ defmodule Supavisor.DbHandler do %{payload: {:authentication_server_first_message, server_first}}, {ps, _} when data.auth.require_user == false -> nonce = data.nonce - server_first_parts = H.parse_server_first(server_first, nonce) + server_first_parts = Helpers.parse_server_first(server_first, nonce) {client_final_message, server_proof} = - H.get_client_final( + Helpers.get_client_final( :auth_query, data.auth.secrets.(), server_first_parts, @@ -229,12 +225,12 @@ defmodule Supavisor.DbHandler do digest = if data.auth.method == :password do - H.md5([data.auth.password.(), data.auth.user]) + Helpers.md5([data.auth.password.(), data.auth.user]) else data.auth.secrets.().secret end - payload = ["md5", H.md5([digest, salt]), 0] + payload = ["md5", Helpers.md5([digest, salt]), 0] bin = [?p, <>, payload] :ok = sock_send(data.sock, bin) {ps, :authentication_md5} @@ -296,7 +292,7 @@ defmodule Supavisor.DbHandler do end def handle_event(:internal, :check_buffer, :idle, %{reply: {from, pid}} = data) do - :ok = H.controlling_process(data.sock, pid) + :ok = Helpers.controlling_process(data.sock, pid) reply = {:reply, from, {:ok, data.sock}} {:next_state, :busy, %{data | reply: nil}, reply} end @@ -356,7 +352,7 @@ defmodule Supavisor.DbHandler do :continue end - :ok = Client.client_cast(data.caller, bin, resp) + :ok = ClientHandler.client_cast(data.caller, bin, resp) if resp != :continue do {_, stats} = Telem.network_usage(:db, data.sock, data.id, data.stats) @@ -369,7 +365,7 @@ defmodule Supavisor.DbHandler do def handle_event(:info, {proto, _, bin}, _, %{caller: caller} = data) when is_pid(caller) and proto in @proto do Logger.debug("DbHandler: Got write replica message #{inspect(bin)}") - HH.setopts(data.sock, active: :once) + HandlerHelpers.setopts(data.sock, active: :once) # check if the response ends with "ready for query" ready = check_ready(bin) sent = data.sent || 0 @@ -382,12 +378,12 @@ defmodule Supavisor.DbHandler do _ -> {:client_call, :continue} end - :ok = apply(Client, send_via, [data.caller, bin, progress]) + :ok = apply(ClientHandler, send_via, [data.caller, bin, progress]) case progress do :ready_for_query -> {_, stats} = Telem.network_usage(:db, data.sock, data.id, data.stats) - HH.setopts(data.sock, active: true) + HandlerHelpers.setopts(data.sock, active: true) {:next_state, :idle, %{data | stats: stats, caller: handler_caller(data), sent: false}, {:next_event, :internal, :check_anon_buffer}} @@ -406,7 +402,7 @@ defmodule Supavisor.DbHandler do def handle_event({:call, from}, {:tcp_owner, pid}, state, %{sock: sock} = data) do if state in [:idle, :busy] do - :ok = H.controlling_process(data.sock, pid) + :ok = Helpers.controlling_process(data.sock, pid) reply = {:reply, from, {:ok, sock}} {:keep_state, data, reply} else @@ -520,7 +516,7 @@ defmodule Supavisor.DbHandler do ) end - @spec try_ssl_handshake(S.tcp_sock(), map) :: {:ok, S.sock()} | {:error, term()} + @spec try_ssl_handshake(Supavisor.tcp_sock(), map) :: {:ok, Supavisor.sock()} | {:error, term()} defp try_ssl_handshake(sock, %{upstream_ssl: true} = auth) do case sock_send(sock, Server.ssl_request()) do :ok -> ssl_recv(sock, auth) @@ -530,7 +526,7 @@ defmodule Supavisor.DbHandler do defp try_ssl_handshake(sock, _), do: {:ok, sock} - @spec ssl_recv(S.tcp_sock(), map) :: {:ok, S.ssl_sock()} | {:error, term} + @spec ssl_recv(Supavisor.tcp_sock(), map) :: {:ok, Supavisor.ssl_sock()} | {:error, term} defp ssl_recv({:gen_tcp, sock} = s, auth) do case :gen_tcp.recv(sock, 1, 15_000) do {:ok, <>} -> @@ -544,7 +540,8 @@ defmodule Supavisor.DbHandler do end end - @spec ssl_connect(S.tcp_sock(), map, pos_integer) :: {:ok, S.ssl_sock()} | {:error, term} + @spec ssl_connect(Supavisor.tcp_sock(), map, pos_integer) :: + {:ok, Supavisor.ssl_sock()} | {:error, term} defp ssl_connect({:gen_tcp, sock}, auth, timeout \\ 5000) do opts = case auth.upstream_verify do @@ -569,7 +566,7 @@ defmodule Supavisor.DbHandler do end end - @spec send_startup(S.sock(), map()) :: :ok | {:error, term} + @spec send_startup(Supavisor.sock(), map()) :: :ok | {:error, term} defp send_startup(sock, auth) do user = get_user(auth) @@ -583,12 +580,12 @@ defmodule Supavisor.DbHandler do sock_send(sock, msg) end - @spec sock_send(S.sock(), iodata) :: :ok | {:error, term} + @spec sock_send(Supavisor.sock(), iodata) :: :ok | {:error, term} defp sock_send({mod, sock}, data) do mod.send(sock, data) end - @spec activate(S.sock()) :: :ok | {:error, term} + @spec activate(Supavisor.sock()) :: :ok | {:error, term} defp activate({:gen_tcp, sock}) do :inet.setopts(sock, active: true) end diff --git a/lib/supavisor/handler_helpers.ex b/lib/supavisor/handler_helpers.ex index a7b58496..91f8055e 100644 --- a/lib/supavisor/handler_helpers.ex +++ b/lib/supavisor/handler_helpers.ex @@ -2,36 +2,33 @@ defmodule Supavisor.HandlerHelpers do @moduledoc false alias Phoenix.PubSub - alias Supavisor, as: S alias Supavisor.Protocol.Server - @spec sock_send(S.sock(), iodata()) :: :ok | {:error, term()} + @spec sock_send(Supavisor.sock(), iodata()) :: :ok | {:error, term()} def sock_send({mod, sock}, data) do mod.send(sock, data) end - @spec sock_close(S.sock() | nil | {any(), nil}) :: :ok | {:error, term()} + @spec sock_close(Supavisor.sock() | nil | {any(), nil}) :: :ok | {:error, term()} def sock_close(nil), do: :ok def sock_close({_, nil}), do: :ok - def sock_close({mod, sock}) do - mod.close(sock) - end + def sock_close({mod, sock}), do: mod.close(sock) - @spec setopts(S.sock(), term()) :: :ok | {:error, term()} + @spec setopts(Supavisor.sock(), term()) :: :ok | {:error, term()} def setopts({mod, sock}, opts) do mod = if mod == :gen_tcp, do: :inet, else: mod mod.setopts(sock, opts) end - @spec active_once(S.sock()) :: :ok | {:error, term} + @spec active_once(Supavisor.sock()) :: :ok | {:error, term} def active_once(sock), do: setopts(sock, active: :once) - @spec activate(S.sock()) :: :ok | {:error, term} + @spec activate(Supavisor.sock()) :: :ok | {:error, term} def activate(sock), do: setopts(sock, active: true) - @spec try_ssl_handshake(S.tcp_sock(), boolean) :: - {:ok, S.sock()} | {:error, term()} + @spec try_ssl_handshake(Supavisor.tcp_sock(), boolean) :: + {:ok, Supavisor.sock()} | {:error, term()} def try_ssl_handshake(sock, true) do case sock_send(sock, Server.ssl_request()) do :ok -> ssl_recv(sock) @@ -41,7 +38,7 @@ defmodule Supavisor.HandlerHelpers do def try_ssl_handshake(sock, false), do: {:ok, sock} - @spec ssl_recv(S.tcp_sock()) :: {:ok, S.ssl_sock()} | {:error, term} + @spec ssl_recv(Supavisor.tcp_sock()) :: {:ok, Supavisor.ssl_sock()} | {:error, term} def ssl_recv({:gen_tcp, sock} = s) do case :gen_tcp.recv(sock, 1, 15_000) do {:ok, <>} -> ssl_connect(s) @@ -50,8 +47,8 @@ defmodule Supavisor.HandlerHelpers do end end - @spec ssl_connect(S.tcp_sock(), pos_integer) :: - {:ok, S.ssl_sock()} | {:error, term} + @spec ssl_connect(Supavisor.tcp_sock(), pos_integer) :: + {:ok, Supavisor.ssl_sock()} | {:error, term} def ssl_connect({:gen_tcp, sock}, timeout \\ 5000) do opts = [verify: :verify_none] @@ -61,13 +58,13 @@ defmodule Supavisor.HandlerHelpers do end end - @spec send_error(S.sock(), String.t(), String.t()) :: :ok | {:error, term()} + @spec send_error(Supavisor.sock(), String.t(), String.t()) :: :ok | {:error, term()} def send_error(sock, code, message) do data = Server.error_message(code, message) sock_send(sock, data) end - @spec try_get_sni(S.sock()) :: String.t() | nil + @spec try_get_sni(Supavisor.sock()) :: String.t() | nil def try_get_sni({:ssl, sock}) do case :ssl.connection_information(sock, [:sni_hostname]) do {:ok, [sni_hostname: sni]} -> List.to_string(sni) @@ -162,7 +159,7 @@ defmodule Supavisor.HandlerHelpers do [] end - @spec addr_from_sock(S.sock()) :: {:ok, :inet.ip_address()} | :error + @spec addr_from_sock(Supavisor.sock()) :: {:ok, :inet.ip_address()} | :error def addr_from_sock({:gen_tcp, port}) do case :inet.peername(port) do {:ok, {:local, _}} -> diff --git a/lib/supavisor_web/controllers/tenant_controller.ex b/lib/supavisor_web/controllers/tenant_controller.ex index 3a5d867f..bf23ef83 100644 --- a/lib/supavisor_web/controllers/tenant_controller.ex +++ b/lib/supavisor_web/controllers/tenant_controller.ex @@ -4,8 +4,7 @@ defmodule SupavisorWeb.TenantController do require Logger - alias Supavisor.Helpers, as: H - alias Supavisor.{Tenants, Repo} + alias Supavisor.{Tenants, Repo, Helpers} alias Tenants.Tenant, as: TenantModel alias SupavisorWeb.OpenApiSchemas.{ @@ -94,7 +93,7 @@ defmodule SupavisorWeb.TenantController do "external_id" => id, "tenant" => %{"upstream_tls_ca" => "-----BEGIN" <> _ = upstream_tls_ca} = tenant_params }) do - case H.cert_to_bin(upstream_tls_ca) do + case Helpers.cert_to_bin(upstream_tls_ca) do {:ok, bin} -> update(conn, %{ "external_id" => id, @@ -112,7 +111,7 @@ defmodule SupavisorWeb.TenantController do def update(conn, %{"external_id" => id, "tenant" => params}) do Logger.info("Delete cache dist #{id}: #{inspect(Supavisor.del_all_cache_dist(id))}") - cert = H.upstream_cert(params["upstream_tls_ca"]) + cert = Helpers.upstream_cert(params["upstream_tls_ca"]) if params["upstream_ssl"] && params["upstream_verify"] == "peer" && !cert do conn @@ -123,7 +122,7 @@ defmodule SupavisorWeb.TenantController do else case Tenants.get_tenant_by_external_id(id) do nil -> - case H.check_creds_get_ver(params) do + case Helpers.check_creds_get_ver(params) do {:error, reason} -> conn |> put_status(400) From 503b653bd07469ab796b408ebab6b0cd18b525ce Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 2 Aug 2024 15:06:30 +0200 Subject: [PATCH 31/97] fix: better handle application name (#412) --- lib/supavisor/handlers/proxy/client.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/supavisor/handlers/proxy/client.ex b/lib/supavisor/handlers/proxy/client.ex index 72c12037..613c6c8c 100644 --- a/lib/supavisor/handlers/proxy/client.ex +++ b/lib/supavisor/handlers/proxy/client.ex @@ -739,10 +739,10 @@ defmodule Supavisor.Handlers.Proxy.Client do max_len = 64 suffix_len = 14 - if String.length(name) <= max_len - suffix_len do + if byte_size(name) <= max_len - suffix_len do name <> suffix else - truncated_name = String.slice(name, 0, max_len - suffix_len - 3) + truncated_name = binary_slice(name, 0, max_len - suffix_len - 3) truncated_name <> "..." <> suffix end end From a19ecaffc5997b6541472fa69be8ee514ed49f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Fri, 2 Aug 2024 15:31:06 +0200 Subject: [PATCH 32/97] chore(native/pgparser): refactor code (#410) --- native/.gitignore | 1 + native/{pgparser => }/Cargo.lock | 63 ++++++++++++++++++------------- native/Cargo.toml | 5 +++ native/pgparser/.gitignore | 1 - native/pgparser/Cargo.toml | 2 +- native/pgparser/src/lib.rs | 35 +++++------------ test/supavisor/pg_parser_test.exs | 5 ++- 7 files changed, 57 insertions(+), 55 deletions(-) create mode 100644 native/.gitignore rename native/{pgparser => }/Cargo.lock (92%) create mode 100644 native/Cargo.toml delete mode 100644 native/pgparser/.gitignore diff --git a/native/.gitignore b/native/.gitignore new file mode 100644 index 00000000..eb5a316c --- /dev/null +++ b/native/.gitignore @@ -0,0 +1 @@ +target diff --git a/native/pgparser/Cargo.lock b/native/Cargo.lock similarity index 92% rename from native/pgparser/Cargo.lock rename to native/Cargo.lock index 9e558302..1d1f73e2 100644 --- a/native/pgparser/Cargo.lock +++ b/native/Cargo.lock @@ -36,7 +36,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.69", + "syn 2.0.72", "which", ] @@ -48,15 +48,15 @@ checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bytes" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.0.105" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" [[package]] name = "cexpr" @@ -168,14 +168,20 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" dependencies = [ "equivalent", "hashbrown", ] +[[package]] +name = "inventory" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767" + [[package]] name = "itertools" version = "0.10.5" @@ -211,9 +217,9 @@ checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", "windows-targets", @@ -314,7 +320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.69", + "syn 2.0.72", ] [[package]] @@ -440,31 +446,33 @@ dependencies = [ [[package]] name = "rustler" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d51ae0239c57c3a3e603dd855ace6795078ef33c95c85d397a100ac62ed352" +checksum = "e94bdfa68c0388cbd725f1ca54e975956482c262599e5cced04a903eec918b7f" dependencies = [ + "inventory", "rustler_codegen", "rustler_sys", ] [[package]] name = "rustler_codegen" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27061f1a2150ad64717dca73902678c124b0619b0d06563294df265bc84759e1" +checksum = "996dc019acb78b91b4e0c1bd6fa2cd509a835d309de762dc15213b97eac399da" dependencies = [ "heck 0.5.0", + "inventory", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.72", ] [[package]] name = "rustler_sys" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2062df0445156ae93cf695ef38c00683848d956b30507592143c01fe8fb52fda" +checksum = "3914a75a147934353c3772a77b774c79fdf80ba84e8347f52a50df0c164aaff2" dependencies = [ "regex", "unreachable", @@ -493,16 +501,17 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.72", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -526,9 +535,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.69" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201fcda3845c23e8212cd466bfebf0bd20694490fc0356ae8e428e0824a915a6" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -549,22 +558,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.72", ] [[package]] diff --git a/native/Cargo.toml b/native/Cargo.toml new file mode 100644 index 00000000..43a9c690 --- /dev/null +++ b/native/Cargo.toml @@ -0,0 +1,5 @@ +[workspace] +resolver = "2" +members = [ + "pgparser" +] diff --git a/native/pgparser/.gitignore b/native/pgparser/.gitignore deleted file mode 100644 index ea8c4bf7..00000000 --- a/native/pgparser/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/native/pgparser/Cargo.toml b/native/pgparser/Cargo.toml index 6c598da2..f4d5923c 100644 --- a/native/pgparser/Cargo.toml +++ b/native/pgparser/Cargo.toml @@ -9,5 +9,5 @@ path = "src/lib.rs" crate-type = ["cdylib"] [dependencies] -rustler = "0.33.0" +rustler = "0.34.0" pg_query = "5.1.0" diff --git a/native/pgparser/src/lib.rs b/native/pgparser/src/lib.rs index 83f2d795..18c3aee2 100644 --- a/native/pgparser/src/lib.rs +++ b/native/pgparser/src/lib.rs @@ -1,28 +1,13 @@ -use rustler::{Atom, Error as RustlerError, NifTuple}; - -mod atoms { - rustler::atoms! { - ok, - error, - } -} - -#[derive(NifTuple)] -struct Response { - status: Atom, - message: Vec -} - #[rustler::nif] -fn statement_types(query: &str) -> Result { - let result = pg_query::parse(&query); - - if let Ok(result) = result { - let message = result.statement_types().into_iter().map(|s| s.to_string()).collect(); - return Ok(Response{status: atoms::ok(), message}); - } else { - return Err(RustlerError::Term(Box::new("Error parsing query"))); - } +fn statement_types(query: &str) -> Result, String> { + let result = pg_query::parse(query).map_err(|_| "Error parsing query")?; + + let message = result + .statement_types() + .into_iter() + .map(Into::into) + .collect(); + Ok(message) } -rustler::init!("Elixir.Supavisor.PgParser", [statement_types]); +rustler::init!("Elixir.Supavisor.PgParser"); diff --git a/test/supavisor/pg_parser_test.exs b/test/supavisor/pg_parser_test.exs index 9209cb8d..54502312 100644 --- a/test/supavisor/pg_parser_test.exs +++ b/test/supavisor/pg_parser_test.exs @@ -1,4 +1,7 @@ defmodule Supavisor.PgParserTest do use ExUnit.Case, async: true - doctest Supavisor.PgParser + + @subject Supavisor.PgParser + + doctest @subject end From f8d5f091d4abd38836e682ebb7c5d57935975850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Fri, 2 Aug 2024 20:53:53 +0200 Subject: [PATCH 33/97] chore: update Erlang version in Flake (#414) --- .gitignore | 1 + flake.nix | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index cb558239..a9065a53 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ supavisor-*.tar.gz priv/native/* *.bggg +/.pre-commit-config.yaml diff --git a/flake.nix b/flake.nix index 0bf40e43..a7bfc0c0 100644 --- a/flake.nix +++ b/flake.nix @@ -39,7 +39,7 @@ devenv-up = self'.devShells.default.config.procfileScript; supavisor = let - erl = pkgs.beam_nox.packages.erlang_26; + erl = pkgs.beam_nox.packages.erlang_27; in erl.callPackage ./nix/package.nix {}; @@ -50,15 +50,25 @@ inherit inputs pkgs; modules = [ + { + pre-commit.hooks = { + alejandra.enable = true; + typos.enable = true; + }; + } { languages.elixir = { enable = true; - package = pkgs.beam.packages.erlang_26.elixir_1_17; + package = pkgs.beam.packages.erlang_27.elixir_1_17; }; packages = [ pkgs.lexical ]; + pre-commit.hooks = { + # credo.enable = true; + }; + # env.DYLD_INSERT_LIBRARIES = "${pkgs.mimalloc}/lib/libmimalloc.dylib"; } { From c1880da51a9228e187cef45a7b0bb7248458ffa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Fri, 2 Aug 2024 20:54:18 +0200 Subject: [PATCH 34/97] test: reduce amount of error log messages during tests (#415) --- config/test.exs | 3 +-- lib/supavisor/handlers/proxy/client.ex | 5 ++++- lib/supavisor/monitoring/prom_ex.ex | 2 +- test/integration/proxy_test.exs | 13 +++++++++---- test/supavisor/prom_ex_test.exs | 2 +- test/support/fixtures/single_connection.ex | 1 + test/test_helper.exs | 2 +- 7 files changed, 18 insertions(+), 10 deletions(-) diff --git a/config/test.exs b/config/test.exs index 0f7fa4b4..aea9b4fa 100644 --- a/config/test.exs +++ b/config/test.exs @@ -42,8 +42,7 @@ config :supavisor, Supavisor.Vault, # Print only warnings and errors during test config :logger, :console, - level: :info, - format: "$time [$level] $message $metadata\n", + level: :error, metadata: [:error_code, :file, :line, :pid, :project, :user, :mode] # Initialize plugs at runtime for faster test compilation diff --git a/lib/supavisor/handlers/proxy/client.ex b/lib/supavisor/handlers/proxy/client.ex index 613c6c8c..7bed2727 100644 --- a/lib/supavisor/handlers/proxy/client.ex +++ b/lib/supavisor/handlers/proxy/client.ex @@ -56,7 +56,10 @@ defmodule Supavisor.Handlers.Proxy.Client do auth: auth, backend_key_data: b }) do - :ok = HandlerHelpers.cancel_query(~c"#{auth.host}", auth.port, auth.ip_ver, b.pid, b.key) + if b != %{} do + :ok = HandlerHelpers.cancel_query(~c"#{auth.host}", auth.port, auth.ip_ver, b.pid, b.key) + end + :keep_state_and_data end diff --git a/lib/supavisor/monitoring/prom_ex.ex b/lib/supavisor/monitoring/prom_ex.ex index 8e3dc78d..01784463 100644 --- a/lib/supavisor/monitoring/prom_ex.ex +++ b/lib/supavisor/monitoring/prom_ex.ex @@ -155,7 +155,7 @@ defmodule Supavisor.Monitoring.PromEx do |> String.trim() if value != cleaned do - Logger.error("Tag validation: #{inspect(value)} / #{inspect(cleaned)}") + Logger.warning("Tag validation: #{inspect(value)} / #{inspect(cleaned)}") end "=\"#{cleaned}\"" diff --git a/test/integration/proxy_test.exs b/test/integration/proxy_test.exs index b3031490..86fc467e 100644 --- a/test/integration/proxy_test.exs +++ b/test/integration/proxy_test.exs @@ -8,7 +8,7 @@ defmodule Supavisor.Integration.ProxyTest do @tenant "proxy_tenant1" - setup_all do + setup do db_conf = Application.get_env(:supavisor, Repo) {:ok, proxy} = @@ -205,8 +205,13 @@ defmodule Supavisor.Integration.ProxyTest do Process.flag(:trap_exit, true) db_conf = Application.get_env(:supavisor, Repo) - url = - "postgresql://max_clients.#{@tenant}:#{db_conf[:password]}@#{db_conf[:hostname]}:#{Application.get_env(:supavisor, :proxy_port_transaction)}/postgres?sslmode=disable" + connection_opts = [ + hostname: db_conf[:hostname], + port: Application.get_env(:supavisor, :proxy_port_transaction), + username: "max_clients.#{@tenant}", + database: "postgres", + password: db_conf[:password] + ] assert {:error, {_, @@ -219,7 +224,7 @@ defmodule Supavisor.Integration.ProxyTest do severity: "FATAL", unknown: "FATAL" } - }, _}}} = parse_uri(url) |> single_connection() + }, _}}} = single_connection(connection_opts) end test "change role password", %{origin: origin} do diff --git a/test/supavisor/prom_ex_test.exs b/test/supavisor/prom_ex_test.exs index 246f1417..11e50b30 100644 --- a/test/supavisor/prom_ex_test.exs +++ b/test/supavisor/prom_ex_test.exs @@ -6,7 +6,7 @@ defmodule Supavisor.PromExTest do @tenant "prom_tenant" - setup_all do + setup do db_conf = Application.get_env(:supavisor, Repo) {:ok, proxy} = diff --git a/test/support/fixtures/single_connection.ex b/test/support/fixtures/single_connection.ex index 887717ca..2f3d89b5 100644 --- a/test/support/fixtures/single_connection.ex +++ b/test/support/fixtures/single_connection.ex @@ -2,6 +2,7 @@ defmodule SingleConnection do @moduledoc false alias Postgrex, as: P + @behaviour P.SimpleConnection def connect(conf) do diff --git a/test/test_helper.exs b/test/test_helper.exs index 92ecaa63..20539b66 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -2,5 +2,5 @@ Cachex.start_link(name: Supavisor.Cache) -ExUnit.start() +ExUnit.start(capture_log: true) Ecto.Adapters.SQL.Sandbox.mode(Supavisor.Repo, :auto) From e081f4ded2495ac4df70361526fd9b52afe85373 Mon Sep 17 00:00:00 2001 From: Stas Date: Sat, 3 Aug 2024 11:25:25 +0200 Subject: [PATCH 35/97] feat: disable metrics via environment variable (#411) --- Makefile | 1 + config/config.exs | 3 +- lib/supavisor/application.ex | 17 ++++-- lib/supavisor/handlers/proxy/handler.ex | 4 +- lib/supavisor/monitoring/telem.ex | 81 ++++++++++++++++--------- 5 files changed, 70 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 36811b2f..4dd6a41f 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ dev: SECRET_KEY_BASE="dev" \ CLUSTER_POSTGRES="true" \ DB_POOL_SIZE="5" \ + METRICS_DISABLED="false" \ ERL_AFLAGS="-kernel shell_history enabled +zdbbl 2097151" \ iex --name node1@127.0.0.1 --cookie cookie -S mix run --no-halt diff --git a/config/config.exs b/config/config.exs index c0bc7c94..9fbf27db 100644 --- a/config/config.exs +++ b/config/config.exs @@ -10,7 +10,8 @@ import Config config :supavisor, ecto_repos: [Supavisor.Repo], version: Mix.Project.config()[:version], - env: Mix.env() + env: Mix.env(), + metrics_disabled: System.get_env("METRICS_DISABLED") == "true" # Configures the endpoint config :supavisor, SupavisorWeb.Endpoint, diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index 7e754862..1e1c003e 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -8,6 +8,8 @@ defmodule Supavisor.Application do alias Supavisor.Monitoring.PromEx alias Supavisor.Handlers.Proxy.Handler, as: ProxyHandler + @metrics_disabled Application.compile_env(:supavisor, :metrics_disabled, false) + @impl true def start(_type, _args) do primary_config = :logger.get_primary_config() @@ -58,13 +60,10 @@ defmodule Supavisor.Application do :syn.set_event_handler(Supavisor.SynHandler) :syn.add_node_to_scopes([:tenants]) - PromEx.set_metrics_tags() - topologies = Application.get_env(:libcluster, :topologies) || [] children = [ Supavisor.ErlSysMon, - PromEx, {Registry, keys: :unique, name: Supavisor.Registry.Tenants}, {Registry, keys: :unique, name: Supavisor.Registry.ManagerTables}, {Registry, keys: :unique, name: Supavisor.Registry.PoolPids}, @@ -81,11 +80,21 @@ defmodule Supavisor.Application do child_spec: DynamicSupervisor, strategy: :one_for_one, name: Supavisor.DynamicSupervisor }, Supavisor.Vault, - Supavisor.TenantsMetrics, + # Start the Endpoint (http/https) SupavisorWeb.Endpoint ] + Logger.warning("metrics_disabled is #{inspect(@metrics_disabled)}") + + children = + if not @metrics_disabled do + PromEx.set_metrics_tags() + children ++ [PromEx, Supavisor.TenantsMetrics] + else + children + end + # start Cachex only if the node uses names, this is necessary for test setup children = if node() != :nonode@nohost do diff --git a/lib/supavisor/handlers/proxy/handler.ex b/lib/supavisor/handlers/proxy/handler.ex index 60b848d2..c08045d5 100644 --- a/lib/supavisor/handlers/proxy/handler.ex +++ b/lib/supavisor/handlers/proxy/handler.ex @@ -15,6 +15,8 @@ defmodule Supavisor.Handlers.Proxy.Handler do Handlers.Proxy.Client } + @metrics_disabled Application.compile_env(:supavisor, :metrics_disabled, false) + @sock_closed [:tcp_closed, :ssl_closed] @proto [:tcp, :ssl] @@ -144,7 +146,7 @@ defmodule Supavisor.Handlers.Proxy.Handler do HandlerHelpers.sock_close(data.sock) HandlerHelpers.sock_close(data.db_sock) - if data.id != nil do + if not @metrics_disabled and data.id != nil do case Registry.lookup(Supavisor.Registry.TenantClients, data.id) do clients when clients == [{self(), []}] or clients == [] -> PromEx.remove_metrics(data.id) _ -> :ok diff --git a/lib/supavisor/monitoring/telem.ex b/lib/supavisor/monitoring/telem.ex index 057e64c2..48c11328 100644 --- a/lib/supavisor/monitoring/telem.ex +++ b/lib/supavisor/monitoring/telem.ex @@ -3,65 +3,86 @@ defmodule Supavisor.Monitoring.Telem do require Logger - alias Supavisor, as: S + @metrics_disabled Application.compile_env(:supavisor, :metrics_disabled, false) - @spec network_usage(:client | :db, S.sock(), S.id(), map()) :: {:ok | :error, map()} + defmacro telemetry_execute(event_name, measurements, metadata) do + if not @metrics_disabled do + quote do + :telemetry.execute(unquote(event_name), unquote(measurements), unquote(metadata)) + end + end + end + + defmacro network_usage_disable(do: block) do + if not @metrics_disabled do + block + else + quote do + {:ok, %{recv_oct: 0, send_oct: 0}} + end + end + end + + @spec network_usage(:client | :db, Supavisor.sock(), Supavisor.id(), map()) :: + {:ok | :error, map()} def network_usage(type, {mod, socket}, id, _stats) do - mod = if mod == :ssl, do: :ssl, else: :inet + network_usage_disable do + mod = if mod == :ssl, do: :ssl, else: :inet - case mod.getstat(socket, [:recv_oct, :send_oct]) do - {:ok, [{:recv_oct, recv_oct}, {:send_oct, send_oct}]} -> - stats = %{ - send_oct: send_oct, - recv_oct: recv_oct - } + case mod.getstat(socket, [:recv_oct, :send_oct]) do + {:ok, [{:recv_oct, recv_oct}, {:send_oct, send_oct}]} -> + stats = %{ + send_oct: send_oct, + recv_oct: recv_oct + } - {{ptype, tenant}, user, mode, db_name} = id + {{ptype, tenant}, user, mode, db_name} = id - :telemetry.execute( - [:supavisor, type, :network, :stat], - stats, - %{tenant: tenant, user: user, mode: mode, type: ptype, db_name: db_name} - ) + :telemetry.execute( + [:supavisor, type, :network, :stat], + stats, + %{tenant: tenant, user: user, mode: mode, type: ptype, db_name: db_name} + ) - {:ok, %{}} + {:ok, %{}} - {:error, reason} -> - Logger.error("Failed to get socket stats: #{inspect(reason)}") - {:error, %{}} + {:error, reason} -> + Logger.error("Failed to get socket stats: #{inspect(reason)}") + {:error, %{}} + end end end - @spec pool_checkout_time(integer(), S.id(), :local | :remote) :: :ok + @spec pool_checkout_time(integer(), Supavisor.id(), :local | :remote) :: :ok | nil def pool_checkout_time(time, {{type, tenant}, user, mode, db_name}, same_box) do - :telemetry.execute( + telemetry_execute( [:supavisor, :pool, :checkout, :stop, same_box], %{duration: time}, %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} ) end - @spec client_query_time(integer(), S.id()) :: :ok + @spec client_query_time(integer(), Supavisor.id()) :: :ok | nil def client_query_time(start, {{type, tenant}, user, mode, db_name}) do - :telemetry.execute( + telemetry_execute( [:supavisor, :client, :query, :stop], %{duration: System.monotonic_time() - start}, %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} ) end - @spec client_connection_time(integer(), S.id()) :: :ok + @spec client_connection_time(integer(), Supavisor.id()) :: :ok | nil def client_connection_time(start, {{type, tenant}, user, mode, db_name}) do - :telemetry.execute( + telemetry_execute( [:supavisor, :client, :connection, :stop], %{duration: System.monotonic_time() - start}, %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} ) end - @spec client_join(:ok | :fail, S.id() | any()) :: :ok + @spec client_join(:ok | :fail, Supavisor.id() | any()) :: :ok | nil def client_join(status, {{type, tenant}, user, mode, db_name}) do - :telemetry.execute( + telemetry_execute( [:supavisor, :client, :joins, status], %{}, %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} @@ -75,10 +96,10 @@ defmodule Supavisor.Monitoring.Telem do @spec handler_action( :client_handler | :db_handler, :started | :stopped | :db_connection, - S.id() - ) :: :ok + Supavisor.id() + ) :: :ok | nil def handler_action(handler, action, {{type, tenant}, user, mode, db_name}) do - :telemetry.execute( + telemetry_execute( [:supavisor, handler, action, :all], %{}, %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} From 5b5de8dd0a41879b846e087d48443599c0bcf9ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Wed, 21 Aug 2024 14:57:26 +0200 Subject: [PATCH 36/97] test: update Postgrex to 0.19.0 (#418) --- mix.lock | 4 +-- test/integration/proxy_test.exs | 56 ++++++++++++++------------------- 2 files changed, 26 insertions(+), 34 deletions(-) diff --git a/mix.lock b/mix.lock index 551946e2..d44d6015 100644 --- a/mix.lock +++ b/mix.lock @@ -26,7 +26,7 @@ "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"}, "inet_cidr": {:hex, :inet_cidr, "1.0.8", "d26bb7bdbdf21ae401ead2092bf2bb4bf57fe44a62f5eaa5025280720ace8a40", [:mix], [], "hexpm", "d5b26da66603bb56c933c65214c72152f0de9a6ea53618b56d63302a68f6a90e"}, - "jason": {:hex, :jason, "1.4.3", "d3f984eeb96fe53b85d20e0b049f03e57d075b5acda3ac8d465c969a2536c17b", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "9a90e868927f7c777689baa16d86f4d0e086d968db5c05d917ccff6d443e58a3"}, + "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "joken": {:hex, :joken, "2.6.1", "2ca3d8d7f83bf7196296a3d9b2ecda421a404634bfc618159981a960020480a1", [:mix], [{:jose, "~> 1.11.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "ab26122c400b3d254ce7d86ed066d6afad27e70416df947cdcb01e13a7382e68"}, "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, "jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"}, @@ -60,7 +60,7 @@ "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "poolboy": {:git, "https://github.com/abc3/poolboy.git", "999ec7f5c7282d515020bb058b4832029d6d07bc", [tag: "v0.0.2"]}, - "postgrex": {:hex, :postgrex, "0.18.0", "f34664101eaca11ff24481ed4c378492fed2ff416cd9b06c399e90f321867d7e", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a042989ba1bc1cca7383ebb9e461398e3f89f868c92ce6671feb7ef132a252d1"}, + "postgrex": {:hex, :postgrex, "0.19.0", "f7d50e50cb42e0a185f5b9a6095125a9ab7e4abccfbe2ab820ab9aa92b71dbab", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "dba2d2a0a8637defbf2307e8629cb2526388ba7348f67d04ec77a5d6a72ecfae"}, "prom_ex": {:git, "https://github.com/hauleth/prom_ex.git", "d6b41ffc6ba77a1dbbbf601eaa76af15a066c101", [branch: "ft/add-peep-storage"]}, "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, "recon": {:hex, :recon, "2.5.5", "c108a4c406fa301a529151a3bb53158cadc4064ec0c5f99b03ddb8c0e4281bdf", [:mix, :rebar3], [], "hexpm", "632a6f447df7ccc1a4a10bdcfce71514412b16660fe59deca0fcf0aa3c054404"}, diff --git a/test/integration/proxy_test.exs b/test/integration/proxy_test.exs index 86fc467e..1b90d1b1 100644 --- a/test/integration/proxy_test.exs +++ b/test/integration/proxy_test.exs @@ -61,11 +61,9 @@ defmodule Supavisor.Integration.ProxyTest do "postgresql://#{db_conf[:username] <> "." <> @tenant}:no_pass@#{db_conf[:hostname]}:#{Application.get_env(:supavisor, :proxy_port_transaction)}/postgres" assert {:error, - {_, - {:stop, - %Postgrex.Error{ - message: "error received in SCRAM server final message: \"Wrong password\"" - }, _}}} = parse_uri(url) |> single_connection() + %Postgrex.Error{ + message: "error received in SCRAM server final message: \"Wrong password\"" + }} = parse_uri(url) |> single_connection() end test "insert", %{proxy: proxy, origin: origin} do @@ -214,17 +212,15 @@ defmodule Supavisor.Integration.ProxyTest do ] assert {:error, - {_, - {:stop, - %Postgrex.Error{ - postgres: %{ - code: :internal_error, - message: "Max client connections reached", - pg_code: "XX000", - severity: "FATAL", - unknown: "FATAL" - } - }, _}}} = single_connection(connection_opts) + %Postgrex.Error{ + postgres: %{ + code: :internal_error, + message: "Max client connections reached", + pg_code: "XX000", + severity: "FATAL", + unknown: "FATAL" + } + }} = single_connection(connection_opts) end test "change role password", %{origin: origin} do @@ -248,11 +244,9 @@ defmodule Supavisor.Integration.ProxyTest do :timer.sleep(1000) assert {:error, - {_, - {:stop, - %Postgrex.Error{ - message: "error received in SCRAM server final message: \"Wrong password\"" - }, _}}} = parse_uri(new_pass) |> single_connection() + %Postgrex.Error{ + message: "error received in SCRAM server final message: \"Wrong password\"" + }} = parse_uri(new_pass) |> single_connection() {:ok, pid} = parse_uri(new_pass) |> single_connection() assert [%Postgrex.Result{rows: [["1"]]}] = P.SimpleConnection.call(pid, {:query, "select 1;"}) @@ -266,17 +260,15 @@ defmodule Supavisor.Integration.ProxyTest do "postgresql://user\"user.#{@tenant}:#{db_conf[:password]}@#{db_conf[:hostname]}:#{Application.get_env(:supavisor, :proxy_port_transaction)}/postgres\\\\\\\\\"\\" assert {:error, - {_, - {:stop, - %Postgrex.Error{ - postgres: %{ - code: :internal_error, - message: "Authentication error, reason: \"Invalid format for user or db_name\"", - pg_code: "XX000", - severity: "FATAL", - unknown: "FATAL" - } - }, _}}} = parse_uri(url) |> single_connection() + %Postgrex.Error{ + postgres: %{ + code: :internal_error, + message: "Authentication error, reason: \"Invalid format for user or db_name\"", + pg_code: "XX000", + severity: "FATAL", + unknown: "FATAL" + } + }} = parse_uri(url) |> single_connection() end defp single_connection(db_conf, c_port \\ nil) when is_list(db_conf) do From e841d8401bc703fddbfc640e6d989b3454181cc8 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 21 Aug 2024 17:16:07 +0200 Subject: [PATCH 37/97] feat: directly send to socket (#419) * feat: directly send to socket * replace cachex.stream to ets.foldl in Supavisor.del_cache_entries --- Makefile | 8 +- config/runtime.exs | 1 + config/test.exs | 1 + lib/supavisor.ex | 34 +- lib/supavisor/application.ex | 6 +- lib/supavisor/client_handler.ex | 308 ++++++---- lib/supavisor/db_handler.ex | 499 +++++++-------- lib/supavisor/handlers/proxy/client.ex | 787 ------------------------ lib/supavisor/handlers/proxy/db.ex | 298 --------- lib/supavisor/handlers/proxy/handler.ex | 156 ----- test/supavisor/db_handler_test.exs | 69 +-- 11 files changed, 444 insertions(+), 1723 deletions(-) delete mode 100644 lib/supavisor/handlers/proxy/client.ex delete mode 100644 lib/supavisor/handlers/proxy/db.ex delete mode 100644 lib/supavisor/handlers/proxy/handler.ex diff --git a/Makefile b/Makefile index 4dd6a41f..11f3918c 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ dev.node2: CLUSTER_POSTGRES="true" \ PROXY_PORT_SESSION="5442" \ PROXY_PORT_TRANSACTION="6553" \ + PROXY_PORT="5402" \ NODE_IP=localhost \ ERL_AFLAGS="-kernel shell_history enabled" \ iex --name node2@127.0.0.1 --cookie cookie -S mix phx.server @@ -106,13 +107,12 @@ dev_start_rel: _build/prod/rel/supavisor/bin/supavisor start_iex prod_rel: - rm -rf _build/prod && \ - MIX_ENV=prod mix compile && \ - MIX_ENV=prod mix release supavisor + MIX_ENV=prod METRICS_DISABLED=true mix compile && \ + MIX_ENV=prod METRICS_DISABLED=true mix release supavisor prod_start_rel: MIX_ENV=prod \ - NODE_NAME=node1 \ + NODE_NAME="localhost" \ VAULT_ENC_KEY="aHD8DZRdk2emnkdktFZRh3E9RNg4aOY7" \ API_JWT_SECRET=dev \ METRICS_JWT_SECRET=dev \ diff --git a/config/runtime.exs b/config/runtime.exs index e3ef8dab..ac72e94b 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -151,6 +151,7 @@ if config_env() != :test do proxy_port_transaction: System.get_env("PROXY_PORT_TRANSACTION", "6543") |> String.to_integer(), proxy_port_session: System.get_env("PROXY_PORT_SESSION", "5432") |> String.to_integer(), + proxy_port: System.get_env("PROXY_PORT", "5412") |> String.to_integer(), prom_poll_rate: System.get_env("PROM_POLL_RATE", "15000") |> String.to_integer(), global_upstream_ca: upstream_ca, global_downstream_cert: downstream_cert, diff --git a/config/test.exs b/config/test.exs index aea9b4fa..0ba015ff 100644 --- a/config/test.exs +++ b/config/test.exs @@ -8,6 +8,7 @@ config :supavisor, jwt_claim_validators: %{}, proxy_port_session: System.get_env("PROXY_PORT_SESSION", "7653") |> String.to_integer(), proxy_port_transaction: System.get_env("PROXY_PORT_TRANSACTION", "7654") |> String.to_integer(), + proxy_port: System.get_env("PROXY_PORT", "5412") |> String.to_integer(), secondary_proxy_port: 7655, secondary_http: 4003, prom_poll_rate: 500, diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 8324552c..6a3c2590 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -1,7 +1,6 @@ defmodule Supavisor do @moduledoc false require Logger - import Cachex.Spec alias Supavisor.{Manager, Helpers, Tenants} @type sock :: tcp_sock() | ssl_sock() @@ -9,7 +8,7 @@ defmodule Supavisor do @type tcp_sock :: {:gen_tcp, :gen_tcp.socket()} @type workers :: %{manager: pid, pool: pid} @type secrets :: {:password | :auth_query, fun()} - @type mode :: :transaction | :session | :native + @type mode :: :transaction | :session | :native | :proxy @type id :: {{:single | :cluster, String.t()}, String.t(), mode, String.t()} @type subscribe_opts :: %{workers: workers, ps: list, idle_timeout: integer} @@ -156,17 +155,24 @@ defmodule Supavisor do [%{inspect(key) => inspect(result)} | acc] end - Supavisor.Cache - |> Cachex.stream!() - |> Enum.reduce([], fn entry(key: key), acc -> - case key do - {:metrics, ^tenant} -> del.(key, acc) - {:secrets, ^tenant, _} -> del.(key, acc) - {:user_cache, _, _, ^tenant, _} -> del.(key, acc) - {:tenant_cache, ^tenant, _} -> del.(key, acc) - _ -> acc - end - end) + :ets.foldl( + fn + {:entry, key, _, _, _result}, acc -> + case key do + {:metrics, ^tenant} -> del.(key, acc) + {:secrets, ^tenant, _} -> del.(key, acc) + {:user_cache, _, _, ^tenant, _} -> del.(key, acc) + {:tenant_cache, ^tenant, _} -> del.(key, acc) + _ -> acc + end + + other, acc -> + Logger.error("Unknown key: #{inspect(other)}") + acc + end, + [], + Supavisor.Cache + ) end @spec del_all_cache_dist(String.t(), pos_integer()) :: [map()] @@ -370,7 +376,7 @@ defmodule Supavisor do socket_opts: [port: 0, keepalive: true] } - handler = Supavisor.Handlers.Proxy.Handler + handler = Supavisor.ClientHandler args = Map.put(args, :local, true) with {:ok, pid} <- :ranch.start_listener(args.id, :ranch_tcp, opts, handler, args) do diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index 1e1c003e..a5ae9ac8 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -6,7 +6,6 @@ defmodule Supavisor.Application do use Application require Logger alias Supavisor.Monitoring.PromEx - alias Supavisor.Handlers.Proxy.Handler, as: ProxyHandler @metrics_disabled Application.compile_env(:supavisor, :metrics_disabled, false) @@ -32,9 +31,10 @@ defmodule Supavisor.Application do proxy_ports = [ {:pg_proxy_transaction, Application.get_env(:supavisor, :proxy_port_transaction), - :transaction, ProxyHandler}, + :transaction, Supavisor.ClientHandler}, {:pg_proxy_session, Application.get_env(:supavisor, :proxy_port_session), :session, - Supavisor.ClientHandler} + Supavisor.ClientHandler}, + {:pg_proxy, Application.get_env(:supavisor, :proxy_port), :proxy, Supavisor.ClientHandler} ] for {key, port, mode, handler} <- proxy_ports do diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 90b85ab1..e24d6f99 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -10,6 +10,8 @@ defmodule Supavisor.ClientHandler do @behaviour :ranch_protocol @behaviour :gen_statem + @proto [:tcp, :ssl] + @cancel_query_msg <<16::32, 1234::16, 5678::16>> alias Supavisor.{ Tenants, @@ -30,23 +32,32 @@ defmodule Supavisor.ClientHandler do @impl true def callback_mode, do: [:handle_event_function] - def client_cast(pid, bin, status) do - :gen_statem.cast(pid, {:client_cast, bin, status}) - end - - @spec client_call(pid, iodata(), atom()) :: :ok | {:error, term()} - def client_call(pid, bin, status), - do: :gen_statem.call(pid, {:client_call, bin, status}, 30_000) + @spec db_status(pid(), :ready_for_query | :read_sql_error, binary()) :: :ok + def db_status(pid, status, bin), do: :gen_statem.cast(pid, {:db_status, status, bin}) @impl true def init(_), do: :ignore def init(ref, trans, opts) do Process.flag(:trap_exit, true) - Helpers.set_max_heap_size(150) + Helpers.set_max_heap_size(90) {:ok, sock} = :ranch.handshake(ref) - :ok = trans.setopts(sock, active: true) + + :ok = + trans.setopts(sock, + # mode: :binary, + # packet: :raw, + # recbuf: 8192, + # sndbuf: 8192, + # # backlog: 2048, + # send_timeout: 120, + # keepalive: true, + # nodelay: true, + # nopush: true, + active: true + ) + Logger.debug("ClientHandler is: #{inspect(self())}") data = %{ @@ -71,7 +82,9 @@ defmodule Supavisor.ClientHandler do last_query: nil, heartbeat_interval: 0, connection_start: System.monotonic_time(), - log_level: nil + log_level: nil, + db_sock: nil, + auth: %{} } :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) @@ -90,7 +103,7 @@ defmodule Supavisor.ClientHandler do end # cancel request - def handle_event(:info, {_, _, <<16::32, 1234::16, 5678::16, pid::32, key::32>>}, _, _) do + def handle_event(:info, {_, _, <<@cancel_query_msg, pid::32, key::32>>}, _, _) do Logger.debug("ClientHandler: Got cancel query for #{inspect({pid, key})}") :ok = HandlerHelpers.send_cancel_query(pid, key) {:stop, {:shutdown, :cancel_query}} @@ -100,7 +113,7 @@ defmodule Supavisor.ClientHandler do def handle_event(:info, :cancel_query, :busy, data) do key = {data.tenant, data.db_pid} Logger.debug("ClientHandler: Cancel query for #{inspect(key)}") - {_pool, db_pid} = data.db_pid + {_pool, db_pid, _db_sock} = data.db_pid case db_pid_meta(key) do [{^db_pid, meta}] -> @@ -161,7 +174,7 @@ defmodule Supavisor.ClientHandler do not_allowed = ["\"", "\\"] if String.contains?(user, not_allowed) or String.contains?(db_name, not_allowed) do - reason = "Invalid characters in user or db_name" + reason = "Invalid format for user or db_name" Logger.error("ClientHandler: #{inspect(reason)}") Telem.client_join(:fail, data.id) @@ -299,11 +312,9 @@ defmodule Supavisor.ClientHandler do ) msg = - if method == :auth_query_md5 do - Server.error_message("XX000", reason) - else - Server.exchange_message(:final, "e=#{reason}") - end + if method == :auth_query_md5, + do: Server.error_message("XX000", reason), + else: Server.exchange_message(:final, "e=#{reason}") key = {:secrets_check, data.tenant, data.user} @@ -338,20 +349,23 @@ defmodule Supavisor.ClientHandler do {:ok, client_key} -> secrets = - if client_key do - fn -> - Map.put(secrets.(), :client_key, client_key) - end - else - secrets - end + if client_key, + do: fn -> Map.put(secrets.(), :client_key, client_key) end, + else: secrets Logger.debug("ClientHandler: Exchange success") :ok = HandlerHelpers.sock_send(sock, Server.authentication_ok()) Telem.client_join(:ok, data.id) - {:keep_state, %{data | auth_secrets: {method, secrets}}, - {:next_event, :internal, :subscribe}} + auth = Map.merge(data.auth, %{secrets: secrets, method: method}) + + conn_type = + if data.mode == :proxy, + do: :connect_db, + else: :subscribe + + {:keep_state, %{data | auth_secrets: {method, secrets}, auth: auth}, + {:next_event, :internal, conn_type}} end end @@ -360,6 +374,7 @@ defmodule Supavisor.ClientHandler do with {:ok, sup} <- Supavisor.start_dist(data.id, data.auth_secrets, log_level: data.log_level), + true <- if(node(sup) != node() and data.mode == :transaction, do: :proxy, else: true), {:ok, opts} <- Supavisor.subscribe(sup, data.id) do Process.monitor(opts.workers.manager) data = Map.merge(data, opts.workers) @@ -367,11 +382,9 @@ defmodule Supavisor.ClientHandler do data = %{data | db_pid: db_pid, idle_timeout: opts.idle_timeout} next = - if opts.ps == [] do - {:timeout, 10_000, :wait_ps} - else - {:next_event, :internal, {:greetings, opts.ps}} - end + if opts.ps == [], + do: {:timeout, 10_000, :wait_ps}, + else: {:next_event, :internal, {:greetings, opts.ps}} {:keep_state, data, next} else @@ -382,12 +395,48 @@ defmodule Supavisor.ClientHandler do Telem.client_join(:fail, data.id) {:stop, {:shutdown, :max_clients_reached}} + :proxy -> + {:ok, %{port: port, host: host}} = Supavisor.get_pool_ranch(data.id) + + auth = + Map.merge(data.auth, %{ + port: port, + host: to_charlist(host), + ip_version: :inet, + upstream_ssl: false, + upstream_tls_ca: nil, + upstream_verify: nil + }) + + {:keep_state, %{data | auth: auth}, {:next_event, :internal, :connect_db}} + error -> Logger.error("ClientHandler: Subscribe error: #{inspect(error)}") {:keep_state_and_data, {:timeout, 1000, :subscribe}} end end + def handle_event(:internal, :connect_db, _, data) do + Logger.debug("ClientHandler: Trying to connect to DB") + + args = %{ + id: data.id, + auth: data.auth, + user: data.user, + tenant: {:single, data.tenant}, + replica_type: :write, + mode: :proxy, + proxy: true, + log_level: data.log_level, + caller: self(), + client_sock: data.sock + } + + {:ok, db_pid} = DbHandler.start_link(args) + db_sock = :gen_statem.call(db_pid, {:checkout, data.sock, self()}) + {:keep_state, %{data | db_pid: {nil, db_pid, db_sock}, mode: :proxy}} + end + def handle_event(:internal, {:greetings, ps}, _, %{sock: sock} = data) do {header, <> = payload} = Server.backend_key_data() msg = [ps, [header, payload], Server.ready_for_query()] @@ -397,9 +446,8 @@ defmodule Supavisor.ClientHandler do {:next_state, :idle, data, handle_actions(data)} end - def handle_event(:timeout, :subscribe, _, _) do - {:keep_state_and_data, {:next_event, :internal, :subscribe}} - end + def handle_event(:timeout, :subscribe, _, _), + do: {:keep_state_and_data, {:next_event, :internal, :subscribe}} def handle_event(:timeout, :wait_ps, _, data) do Logger.error( @@ -423,43 +471,45 @@ defmodule Supavisor.ClientHandler do # handle Terminate message def handle_event(:info, {proto, _, <>}, :idle, _) - when proto in [:tcp, :ssl] do - Logger.debug("ClientHandler: Terminate received from client") + when proto in @proto do + Logger.error("ClientHandler: Terminate received from client") {:stop, {:shutdown, :terminate_received}} end # handle Sync message def handle_event(:info, {proto, _, <>}, :idle, data) - when proto in [:tcp, :ssl] do - Logger.debug("ClientHandler: Receive sync") + when proto in @proto do + Logger.error("ClientHandler: Receive sync") :ok = HandlerHelpers.sock_send(data.sock, Server.ready_for_query()) {:keep_state_and_data, handle_actions(data)} end def handle_event(:info, {proto, _, <> = msg}, _, data) - when proto in [:tcp, :ssl] do - Logger.debug("ClientHandler: Receive sync while not idle") - {_, db_pid} = data.db_pid - DbHandler.cast(db_pid, self(), msg) + when proto in @proto do + Logger.error("ClientHandler: Receive sync while not idle") + :ok = HandlerHelpers.sock_send(elem(data.db_pid, 2), msg) :keep_state_and_data end def handle_event(:info, {proto, _, <> = msg}, _, data) - when proto in [:tcp, :ssl] do - Logger.debug("ClientHandler: Receive flush while not idle") - {_, db_pid} = data.db_pid - DbHandler.cast(db_pid, self(), msg) + when proto in @proto do + Logger.error("ClientHandler: Receive flush while not idle") + :ok = HandlerHelpers.sock_send(elem(data.db_pid, 2), msg) :keep_state_and_data end # incoming query with a single pool def handle_event(:info, {proto, _, bin}, :idle, %{pool: pid} = data) when is_binary(bin) and is_pid(pid) do - ts = System.monotonic_time() + Logger.debug("ClientHandler: Receive query #{inspect(bin)}") db_pid = db_checkout(:both, :on_query, data) - handle_prepared_statements(db_pid, bin, data) - {:next_state, :busy, %{data | db_pid: db_pid, query_start: ts}, + {:next_state, :busy, %{data | db_pid: db_pid, query_start: System.monotonic_time()}, + {:next_event, :internal, {proto, nil, bin}}} + end + + def handle_event(:info, {proto, _, bin}, _, %{mode: :proxy} = data) do + {:next_state, :busy, %{data | query_start: System.monotonic_time()}, {:next_event, :internal, {proto, nil, bin}}} end @@ -491,33 +541,24 @@ defmodule Supavisor.ClientHandler do end # forward query to db - def handle_event(_, {proto, _, bin}, :busy, data) - when proto in [:tcp, :ssl] do - {_, db_pid} = data.db_pid + def handle_event(_, {proto, _, bin}, :busy, data) when proto in @proto do + # HandlerHelpers.setopts(data.sock, active: :once) - case DbHandler.call(db_pid, self(), bin) do + Logger.debug("ClientHandler: Forward query to db #{inspect(bin)} #{inspect(data.db_pid)}") + + case HandlerHelpers.sock_send(elem(data.db_pid, 2), bin) do :ok -> - Logger.debug("ClientHandler: DbHandler call success") :keep_state_and_data - {:buffering, size} -> - Logger.debug("ClientHandler: DbHandler call buffering #{size}") + error -> + Logger.error("ClientHandler: error while sending query: #{inspect(error)}") - if size > 1_000_000 do - msg = "DbHandler buffer size is too big: #{size}" - Logger.error("ClientHandler: #{msg}") - HandlerHelpers.sock_send(data.sock, Server.error_message("XX000", msg)) - {:stop, {:shutdown, :buffer_size}} - else - Logger.debug("ClientHandler: DbHandler call buffering") - :keep_state_and_data - end + HandlerHelpers.sock_send( + data.sock, + Server.error_message("XX000", "Error while sending query") + ) - {:error, reason} -> - msg = "DbHandler error: #{inspect(reason)}" - Logger.error("ClientHandler: #{msg}") - HandlerHelpers.sock_send(data.sock, Server.error_message("XX000", msg)) - {:stop, {:shutdown, :db_handler_error}} + {:stop, {:shutdown, :send_query_error}} end end @@ -526,9 +567,8 @@ defmodule Supavisor.ClientHandler do {:stop, {:shutdown, :parameter_status_updated}} end - def handle_event(:info, {:parameter_status, ps}, :exchange, _) do - {:keep_state_and_data, {:next_event, :internal, {:greetings, ps}}} - end + def handle_event(:info, {:parameter_status, ps}, :exchange, _), + do: {:keep_state_and_data, {:next_event, :internal, {:greetings, ps}}} # client closed connection def handle_event(_, {closed, _}, _, data) @@ -551,14 +591,9 @@ defmodule Supavisor.ClientHandler do ) case {state, reason} do - {_, :shutdown} -> - {:stop, {:shutdown, :manager_shutdown}} - - {:idle, _} -> - {:keep_state_and_data, {:next_event, :internal, :subscribe}} - - {:busy, _} -> - {:stop, {:shutdown, :manager_down}} + {_, :shutdown} -> {:stop, {:shutdown, :manager_shutdown}} + {:idle, _} -> {:keep_state_and_data, {:next_event, :internal, :subscribe}} + {:busy, _} -> {:stop, {:shutdown, :manager_down}} end end @@ -568,32 +603,21 @@ defmodule Supavisor.ClientHandler do end # emulate handle_cast - def handle_event(:cast, {:client_cast, bin, status}, _, data) do - Logger.debug("ClientHandler: --> --> bin #{inspect(byte_size(bin))} bytes") - + def handle_event(:cast, {:db_status, status, bin}, :busy, data) do case status do :ready_for_query -> Logger.debug("ClientHandler: Client is ready") + HandlerHelpers.sock_send(data.sock, bin) db_pid = handle_db_pid(data.mode, data.pool, data.db_pid) {_, stats} = Telem.network_usage(:client, data.sock, data.id, data.stats) Telem.client_query_time(data.query_start, data.id) - :ok = HandlerHelpers.sock_send(data.sock, bin) - actions = handle_actions(data) - {:next_state, :idle, %{data | db_pid: db_pid, stats: stats}, actions} - - :continue -> - Logger.debug("ClientHandler: Client is not ready") - :ok = HandlerHelpers.sock_send(data.sock, bin) - :keep_state_and_data + {:next_state, :idle, %{data | db_pid: db_pid, stats: stats}, handle_actions(data)} :read_sql_error -> - Logger.error( - "ClientHandler: read only sql transaction, rerunning the query to write pool" - ) - + Logger.error("ClientHandler: read only sql transaction, reruning the query to write pool") # release the read pool _ = handle_db_pid(data.mode, data.pool, data.db_pid) @@ -605,12 +629,6 @@ defmodule Supavisor.ClientHandler do end end - # emulate handle_call - def handle_event({:call, from}, {:client_call, bin, _}, _, data) do - Logger.debug("ClientHandler: --> --> bin call #{inspect(byte_size(bin))} bytes") - {:keep_state_and_data, {:reply, from, HandlerHelpers.sock_send(data.sock, bin)}} - end - def handle_event(type, content, state, data) do msg = [ {"type", type}, @@ -651,7 +669,7 @@ defmodule Supavisor.ClientHandler do resp end - Logger.warning( + Logger.debug( "ClientHandler: socket closed with reason #{inspect(reason)}, DbHandler #{inspect({pid, db_info})}" ) @@ -659,7 +677,7 @@ defmodule Supavisor.ClientHandler do end def terminate(reason, _state, _data) do - Logger.warning("ClientHandler: socket closed with reason #{inspect(reason)}") + Logger.debug("ClientHandler: socket closed with reason #{inspect(reason)}") :ok end @@ -737,11 +755,9 @@ defmodule Supavisor.ClientHandler do end defp authenticate_exchange(:password, _secrets, signatures, p) do - if p == signatures.client do - {:ok, nil} - else - {:error, "Wrong password"} - end + if p == signatures.client, + do: {:ok, nil}, + else: {:error, "Wrong password"} end defp authenticate_exchange(:auth_query, secrets, signatures, p) do @@ -755,17 +771,15 @@ defmodule Supavisor.ClientHandler do end defp authenticate_exchange(:auth_query_md5, client_hash, server_hash, salt) do - if "md5" <> Helpers.md5([server_hash, salt]) == client_hash do - {:ok, nil} - else - {:error, "Wrong password"} - end + if "md5" <> Helpers.md5([server_hash, salt]) == client_hash, + do: {:ok, nil}, + else: {:error, "Wrong password"} end @spec db_checkout(:write | :read | :both, :on_connect | :on_query, map) :: {pid, pid} | nil - defp db_checkout(_, _, %{mode: :session, db_pid: {pool, db_pid}}) - when is_pid(pool) and is_pid(db_pid) do - {pool, db_pid} + defp db_checkout(_, _, %{mode: mode, db_pid: {pool, db_pid, db_sock}}) + when is_pid(db_pid) and mode in [:session, :proxy] do + {pool, db_pid, db_sock} end defp db_checkout(_, :on_connect, %{mode: :transaction}), do: nil @@ -783,32 +797,49 @@ defmodule Supavisor.ClientHandler do end defp db_checkout(_, _, data) do - {time, db_pid} = :timer.tc(:poolboy, :checkout, [data.pool, true, data.timeout]) + start = System.monotonic_time(:microsecond) + db_pid = :poolboy.checkout(data.pool, true, data.timeout) Process.link(db_pid) + db_sock = DbHandler.checkout(db_pid, data.sock, self()) same_box = if node(db_pid) == node(), do: :local, else: :remote - Telem.pool_checkout_time(time, data.id, same_box) - {data.pool, db_pid} + Telem.pool_checkout_time(System.monotonic_time(:microsecond) - start, data.id, same_box) + {data.pool, db_pid, db_sock} end @spec handle_db_pid(:transaction, pid(), pid() | nil) :: nil @spec handle_db_pid(:session, pid(), pid()) :: pid() + @spec handle_db_pid(:proxy, pid(), pid()) :: pid() defp handle_db_pid(:transaction, _pool, nil), do: nil - defp handle_db_pid(:transaction, _pool, {pool, db_pid}) do + defp handle_db_pid(:transaction, pool, {_, db_pid, _}) do Process.unlink(db_pid) :poolboy.checkin(pool, db_pid) nil end defp handle_db_pid(:session, _, db_pid), do: db_pid + defp handle_db_pid(:proxy, _, db_pid), do: db_pid defp update_user_data(data, info, user, id, db_name, mode) do proxy_type = - if info.tenant.require_user do - :password - else - :auth_query - end + if info.tenant.require_user, + do: :password, + else: :auth_query + + auth = %{ + application_name: data[:app_name] || "Supavisor", + database: info.tenant.db_database, + host: to_charlist(info.tenant.db_host), + sni_host: info.tenant.sni_hostname, + ip_version: Helpers.ip_version(info.tenant.ip_version, info.tenant.db_host), + port: info.tenant.db_port, + user: user, + password: info.user.db_password, + require_user: info.tenant.require_user, + upstream_ssl: info.tenant.upstream_ssl, + upstream_tls_ca: info.tenant.upstream_tls_ca, + upstream_verify: info.tenant.upstream_verify + } %{ data @@ -820,7 +851,8 @@ defmodule Supavisor.ClientHandler do id: id, heartbeat_interval: info.tenant.client_heartbeat_interval * 1000, db_name: db_name, - mode: mode + mode: mode, + auth: auth } end @@ -878,17 +910,21 @@ defmodule Supavisor.ClientHandler do ssl_opts: ssl_opts || [] ) - resp = - case Helpers.get_user_secret(conn, tenant.auth_query, db_user) do - {:ok, secret} -> - t = if secret.digest == :md5, do: :auth_query_md5, else: :auth_query - {:ok, {t, fn -> Map.put(secret, :alias, user.db_user_alias) end}} + # kill the postgrex connection if the current process exits unexpectedly + Process.link(conn) - {:error, reason} -> - {:error, reason} + Logger.debug( + "ClientHandler: Connected to db #{tenant.db_host} #{tenant.db_port} #{tenant.db_database} #{user.db_user}" + ) + + resp = + with {:ok, secret} <- Helpers.get_user_secret(conn, tenant.auth_query, db_user) do + t = if secret.digest == :md5, do: :auth_query_md5, else: :auth_query + {:ok, {t, fn -> Map.put(secret, :alias, user.db_user_alias) end}} end - GenServer.stop(conn, :normal) + GenServer.stop(conn, :normal, 5_000) + Logger.info("ProxyClient: Get secrets finished") resp end diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 9c36dce6..ed7f71ad 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -15,23 +15,15 @@ defmodule Supavisor.DbHandler do @reconnect_timeout 2_500 @sock_closed [:tcp_closed, :ssl_closed] @proto [:tcp, :ssl] - @async_send_limit 1_000 - def start_link(config) do - :gen_statem.start_link(__MODULE__, config, hibernate_after: 5_000) - end - - @spec call(pid(), pid(), binary()) :: :ok | {:error, any()} | {:buffering, non_neg_integer()} - def call(pid, caller, msg), do: :gen_statem.call(pid, {:db_call, caller, msg}, 15_000) - - @spec cast(pid(), pid(), binary()) :: :ok | {:error, any()} | {:buffering, non_neg_integer()} - def cast(pid, caller, msg), do: :gen_statem.cast(pid, {:db_cast, caller, msg}) + def start_link(config), + do: :gen_statem.start_link(__MODULE__, config, hibernate_after: 5_000) - @spec set_idle(pid()) :: :ok - def set_idle(pid), do: :gen_statem.cast(pid, :set_idle) + def checkout(pid, sock, caller, timeout \\ 15_000), + do: :gen_statem.call(pid, {:checkout, sock, caller}, timeout) - @spec change_socket_owner(pid(), pid()) :: {:ok, Supavisor.sock()} - def change_socket_owner(pid, caller), do: :gen_statem.call(pid, {:tcp_owner, caller}, 15_000) + @spec checkin(pid()) :: :ok + def checkin(pid), do: :gen_statem.cast(pid, :checkin) @spec get_state_and_mode(pid()) :: {:ok, {state, Supavisor.mode()}} | {:error, term()} def get_state_and_mode(pid) do @@ -49,31 +41,35 @@ defmodule Supavisor.DbHandler do def init(args) do Process.flag(:trap_exit, true) Helpers.set_log_level(args.log_level) - Helpers.set_max_heap_size(150) + Helpers.set_max_heap_size(90) {_, tenant} = args.tenant Logger.metadata(project: tenant, user: args.user, mode: args.mode) - data = %{ - id: args.id, - sock: nil, - caller: nil, - sent: false, - auth: args.auth, - user: args.user, - tenant: args.tenant, - buffer: [], - anon_buffer: [], - db_state: nil, - parameter_status: %{}, - nonce: nil, - messages: "", - server_proof: nil, - stats: %{}, - mode: args.mode, - replica_type: args.replica_type, - reply: nil - } + data = + %{ + id: args.id, + sock: nil, + sent: false, + auth: args.auth, + user: args.user, + tenant: args.tenant, + buffer: [], + anon_buffer: [], + db_state: nil, + parameter_status: %{}, + nonce: nil, + messages: "", + server_proof: nil, + stats: %{}, + mode: args.mode, + replica_type: args.replica_type, + reply: nil, + pool: Supavisor.get_local_pool(args.id), + caller: args[:caller] || nil, + client_sock: args[:client_sock] || nil, + proxy: args[:proxy] || false + } Telem.handler_action(:db_handler, :started, args.id) {:ok, :connect, data, {:next_event, :internal, :connect}} @@ -86,13 +82,20 @@ defmodule Supavisor.DbHandler do def handle_event(:internal, _, :connect, %{auth: auth} = data) do Logger.debug("DbHandler: Try to connect to DB") - sock_opts = [ - :binary, - {:packet, :raw}, - {:active, false}, - {:nodelay, true}, - auth.ip_version - ] + sock_opts = + [ + auth.ip_version, + mode: :binary, + packet: :raw, + # recbuf: 8192, + # sndbuf: 8192, + # backlog: 2048, + # send_timeout: 120, + # keepalive: true, + # nopush: true, + nodelay: true, + active: true + ] reconnect_callback = {:keep_state_and_data, {:state_timeout, @reconnect_timeout, :connect}} @@ -104,7 +107,9 @@ defmodule Supavisor.DbHandler do case try_ssl_handshake({:gen_tcp, sock}, auth) do {:ok, sock} -> - case send_startup(sock, auth) do + tenant = if data.proxy, do: Supavisor.tenant(data.id) + + case send_startup(sock, auth, tenant) do :ok -> :ok = activate(sock) {:next_state, :authentication, %{data | sock: sock}} @@ -137,137 +142,28 @@ defmodule Supavisor.DbHandler do dec_pkt = Server.decode(bin) Logger.debug("DbHandler: dec_pkt, #{inspect(dec_pkt, pretty: true)}") - resp = - Enum.reduce(dec_pkt, {%{}, nil}, fn - %{tag: :parameter_status, payload: {k, v}}, {ps, db_state} -> - {Map.put(ps, k, v), db_state} - - %{tag: :ready_for_query, payload: db_state}, {ps, _} -> - {:ready_for_query, ps, db_state} - - %{tag: :backend_key_data, payload: payload}, acc -> - key = self() - conn = %{host: data.auth.host, port: data.auth.port, ip_ver: data.auth.ip_version} - Registry.register(Supavisor.Registry.PoolPids, key, Map.merge(payload, conn)) - Logger.debug("DbHandler: Backend #{inspect(key)} data: #{inspect(payload)}") - acc - - %{payload: {:authentication_sasl_password, methods_b}}, {ps, _} -> - nonce = - case Server.decode_string(methods_b) do - {:ok, req_method, _} -> - Logger.debug("DbHandler: SASL method #{inspect(req_method)}") - nonce = :pgo_scram.get_nonce(16) - user = get_user(data.auth) - client_first = :pgo_scram.get_client_first(user, nonce) - client_first_size = IO.iodata_length(client_first) - - sasl_initial_response = [ - "SCRAM-SHA-256", - 0, - <>, - client_first - ] - - bin = :pgo_protocol.encode_scram_response_message(sasl_initial_response) - :ok = sock_send(data.sock, bin) - nonce - - other -> - Logger.error("DbHandler: Undefined sasl method #{inspect(other)}") - nil - end - - {ps, :authentication_sasl, nonce} - - %{payload: {:authentication_server_first_message, server_first}}, {ps, _} - when data.auth.require_user == false -> - nonce = data.nonce - server_first_parts = Helpers.parse_server_first(server_first, nonce) - - {client_final_message, server_proof} = - Helpers.get_client_final( - :auth_query, - data.auth.secrets.(), - server_first_parts, - nonce, - data.auth.secrets.().user, - "biws" - ) - - bin = :pgo_protocol.encode_scram_response_message(client_final_message) - :ok = sock_send(data.sock, bin) - - {ps, :authentication_server_first_message, server_proof} - - %{payload: {:authentication_server_first_message, server_first}}, {ps, _} -> - nonce = data.nonce - server_first_parts = :pgo_scram.parse_server_first(server_first, nonce) - - {client_final_message, server_proof} = - :pgo_scram.get_client_final( - server_first_parts, - nonce, - data.auth.user, - data.auth.password.() - ) - - bin = :pgo_protocol.encode_scram_response_message(client_final_message) - :ok = sock_send(data.sock, bin) - - {ps, :authentication_server_first_message, server_proof} - - %{payload: {:authentication_server_final_message, _server_final}}, acc -> - acc - - %{payload: {:authentication_md5_password, salt}}, {ps, _} -> - Logger.debug("DbHandler: dec_pkt, #{inspect(dec_pkt, pretty: true)}") - - digest = - if data.auth.method == :password do - Helpers.md5([data.auth.password.(), data.auth.user]) - else - data.auth.secrets.().secret - end - - payload = ["md5", Helpers.md5([digest, salt]), 0] - bin = [?p, <>, payload] - :ok = sock_send(data.sock, bin) - {ps, :authentication_md5} - - %{tag: :error_response, payload: error}, _ -> - {:error_response, error} - - _e, acc -> - acc - end) + resp = Enum.reduce(dec_pkt, %{}, &handle_auth_pkts(&1, &2, data)) case resp do - {_, :authentication_sasl, nonce} -> + {:authentication_sasl, nonce} -> {:keep_state, %{data | nonce: nonce}} - {_, :authentication_server_first_message, server_proof} -> + {:authentication_server_first_message, server_proof} -> {:keep_state, %{data | server_proof: server_proof}} - {_, :authentication_md5} -> + %{authentication_server_final_message: _server_final} -> + :keep_state_and_data + + %{authentication_ok: true} -> + :keep_state_and_data + + :authentication -> + :keep_state_and_data + + :authentication_md5 -> {:keep_state, data} {:error_response, ["SFATAL", "VFATAL", "C28P01", reason, _, _, _]} -> - tenant = Supavisor.tenant(data.id) - - for node <- [node() | Node.list()] do - :erpc.cast(node, fn -> - Cachex.del(Supavisor.Cache, {:secrets, tenant, data.user}) - Cachex.del(Supavisor.Cache, {:secrets_check, tenant, data.user}) - - Registry.dispatch(Supavisor.Registry.TenantClients, data.id, fn entries -> - for {client_handler, _meta} <- entries, - do: send(client_handler, {:disconnect, reason}) - end) - end) - end - - Supavisor.stop(data.id) Logger.error("DbHandler: Auth error #{inspect(reason)}") {:stop, :invalid_password, data} @@ -275,12 +171,19 @@ defmodule Supavisor.DbHandler do Logger.error("DbHandler: Error auth response #{inspect(error)}") {:keep_state, data} - {:ready_for_query, ps, db_state} -> + {:ready_for_query, acc} -> + ps = acc.ps + Logger.debug( - "DbHandler: DB ready_for_query: #{inspect(db_state)} #{inspect(ps, pretty: true)}" + "DbHandler: DB ready_for_query: #{inspect(acc.db_state)} #{inspect(ps, pretty: true)}" ) - Supavisor.set_parameter_status(data.id, ps) + if data.proxy do + bin_ps = Server.encode_parameter_status(ps) + send(data.caller, {:parameter_status, bin_ps}) + else + Supavisor.set_parameter_status(data.id, ps) + end {:next_state, :idle, %{data | parameter_status: ps}, {:next_event, :internal, :check_buffer}} @@ -291,10 +194,9 @@ defmodule Supavisor.DbHandler do end end - def handle_event(:internal, :check_buffer, :idle, %{reply: {from, pid}} = data) do - :ok = Helpers.controlling_process(data.sock, pid) - reply = {:reply, from, {:ok, data.sock}} - {:next_state, :busy, %{data | reply: nil}, reply} + def handle_event(:internal, :check_buffer, :idle, %{reply: from} = data) when from != nil do + Logger.debug("DbHandler: Check buffer") + {:next_state, :busy, %{data | reply: nil}, {:reply, from, data.sock}} end def handle_event(:internal, :check_buffer, :idle, %{buffer: buff, caller: caller} = data) @@ -310,6 +212,8 @@ defmodule Supavisor.DbHandler do # check if it needs to apply queries from the anon buffer def handle_event(:internal, :check_anon_buffer, _, %{anon_buffer: buff, caller: nil} = data) do + Logger.debug("DbHandler: Check anon buffer") + if buff != [] do Logger.debug( "DbHandler: Anon buffer is not empty, try to send #{IO.iodata_length(buff)} bytes" @@ -322,6 +226,11 @@ defmodule Supavisor.DbHandler do {:keep_state, %{data | anon_buffer: []}} end + def handle_event(:internal, :check_anon_buffer, _, _) do + Logger.debug("DbHandler: Anon buffer is empty") + :keep_state_and_data + end + # the process received message from db without linked caller def handle_event(:info, {proto, _, bin}, _, %{caller: nil}) when proto in @proto do Logger.debug("DbHandler: Got db response #{inspect(bin)} when caller was nil") @@ -352,9 +261,8 @@ defmodule Supavisor.DbHandler do :continue end - :ok = ClientHandler.client_cast(data.caller, bin, resp) - if resp != :continue do + :ok = ClientHandler.db_status(data.caller, resp, bin) {_, stats} = Telem.network_usage(:db, data.sock, data.id, data.stats) {:keep_state, %{data | stats: stats, caller: handler_caller(data)}} else @@ -362,34 +270,27 @@ defmodule Supavisor.DbHandler do end end - def handle_event(:info, {proto, _, bin}, _, %{caller: caller} = data) + # forward the message to the client + def handle_event(:info, {proto, _, bin}, _, %{caller: caller, reply: nil} = data) when is_pid(caller) and proto in @proto do - Logger.debug("DbHandler: Got write replica message #{inspect(bin)}") - HandlerHelpers.setopts(data.sock, active: :once) - # check if the response ends with "ready for query" - ready = check_ready(bin) - sent = data.sent || 0 - - {send_via, progress} = - case ready do - {:ready_for_query, :idle} -> {:client_cast, :ready_for_query} - {:ready_for_query, _} -> {:client_cast, :continue} - _ when sent < @async_send_limit -> {:client_cast, :continue} - _ -> {:client_call, :continue} - end - - :ok = apply(ClientHandler, send_via, [data.caller, bin, progress]) + Logger.debug("DbHandler: Got write replica message #{inspect(bin)}") - case progress do - :ready_for_query -> - {_, stats} = Telem.network_usage(:db, data.sock, data.id, data.stats) - HandlerHelpers.setopts(data.sock, active: true) + if String.ends_with?(bin, Server.ready_for_query()) do + {_, stats} = Telem.network_usage(:db, data.sock, data.id, data.stats) - {:next_state, :idle, %{data | stats: stats, caller: handler_caller(data), sent: false}, - {:next_event, :internal, :check_anon_buffer}} + data = + if data.mode == :transaction do + ClientHandler.db_status(data.caller, :ready_for_query, bin) + %{data | stats: stats, caller: nil, client_sock: nil} + else + HandlerHelpers.sock_send(data.client_sock, bin) + %{data | stats: stats} + end - :continue -> - {:keep_state, %{data | sent: sent + 1}} + {:next_state, :idle, data, {:next_event, :internal, :check_anon_buffer}} + else + HandlerHelpers.sock_send(data.client_sock, bin) + :keep_state_and_data end end @@ -400,60 +301,18 @@ defmodule Supavisor.DbHandler do {:next_event, :internal, :check_anon_buffer}} end - def handle_event({:call, from}, {:tcp_owner, pid}, state, %{sock: sock} = data) do - if state in [:idle, :busy] do - :ok = Helpers.controlling_process(data.sock, pid) - reply = {:reply, from, {:ok, sock}} - {:keep_state, data, reply} - else - Logger.debug("DbHandler: TCP owner call when state was #{state}") - {:keep_state, %{data | reply: {from, pid}}} - end - end + def handle_event({:call, from}, {:checkout, sock, caller}, state, data) do + Logger.debug("DbHandler: checkout call when state was #{state}") - def handle_event({:call, from}, {:db_call, caller, bin}, :idle, %{sock: sock} = data) do - reply = {:reply, from, sock_send(sock, bin)} - {:next_state, :busy, %{data | caller: caller}, reply} + # store the reply ref and send it when the state is idle + if state in [:idle, :busy], + do: {:keep_state, %{data | client_sock: sock, caller: caller}, {:reply, from, data.sock}}, + else: {:keep_state, %{data | client_sock: sock, caller: caller, reply: from}} end - def handle_event({:call, from}, {:db_call, caller, bin}, :busy, %{sock: sock} = data) do - reply = {:reply, from, sock_send(sock, bin)} - {:keep_state, %{data | caller: caller}, reply} - end - - def handle_event({:call, from}, {:db_call, caller, bin}, state, %{buffer: buff} = data) do - Logger.debug( - "DbHandler: state #{state} <-- <-- bin #{inspect(byte_size(bin))} bytes, caller: #{inspect(caller)}" - ) - - new_buff = [bin | buff] - reply = {:reply, from, {:buffering, IO.iodata_length(new_buff)}} - {:keep_state, %{data | caller: caller, buffer: new_buff}, reply} - end - - # emulate handle_cast - def handle_event(:cast, {:db_cast, caller, bin}, state, %{sock: sock}) - when state in [:idle, :busy] do - Logger.debug( - "DbHandler: state #{state} <-- <-- bin #{inspect(byte_size(bin))} bytes, cast caller: #{inspect(caller)}" - ) - - sock_send(sock, bin) - :keep_state_and_data - end - - def handle_event(:cast, {:db_cast, caller, bin}, state, %{buffer: buff} = data) do - Logger.debug( - "DbHandler: state #{state} <-- <-- bin #{inspect(byte_size(bin))} bytes, cast caller: #{inspect(caller)}" - ) - - new_buff = [bin | buff] - {:keep_state, %{data | caller: caller, buffer: new_buff}} - end - - def handle_event(:cast, :set_idle, _, data) do - Logger.debug("DbHandler: set_idle") - {:next_state, :idle, data} + def handle_event({:call, from}, :ps, _, data) do + Logger.debug("DbHandler: get parameter status") + {:keep_state_and_data, {:reply, from, data.parameter_status}} end def handle_event(_, {closed, _}, :busy, data) when closed in @sock_closed do @@ -529,14 +388,9 @@ defmodule Supavisor.DbHandler do @spec ssl_recv(Supavisor.tcp_sock(), map) :: {:ok, Supavisor.ssl_sock()} | {:error, term} defp ssl_recv({:gen_tcp, sock} = s, auth) do case :gen_tcp.recv(sock, 1, 15_000) do - {:ok, <>} -> - ssl_connect(s, auth) - - {:ok, <>} -> - {:error, :ssl_not_available} - - {:error, _} = error -> - error + {:ok, <>} -> ssl_connect(s, auth) + {:ok, <>} -> {:error, :ssl_not_available} + {:error, _} = error -> error end end @@ -549,7 +403,8 @@ defmodule Supavisor.DbHandler do [ verify: :verify_peer, cacerts: [auth.upstream_tls_ca], - server_name_indication: auth.host, + # unclear behavior on pg14 + server_name_indication: auth.sni_host || auth.host, customize_hostname_check: [{:match_fun, fn _, _ -> true end}] ] @@ -566,9 +421,10 @@ defmodule Supavisor.DbHandler do end end - @spec send_startup(Supavisor.sock(), map()) :: :ok | {:error, term} - defp send_startup(sock, auth) do - user = get_user(auth) + @spec send_startup(Supavisor.sock(), map(), String.t() | nil) :: :ok | {:error, term} + def send_startup(sock, auth, tenant) do + user = + if is_nil(tenant), do: get_user(auth), else: "#{get_user(auth)}.#{tenant}" msg = :pgo_protocol.encode_startup_message([ @@ -637,4 +493,125 @@ defmodule Supavisor.DbHandler do :continue end end + + @spec handle_auth_pkts(map(), map(), map()) :: any() + defp handle_auth_pkts(%{tag: :parameter_status, payload: {k, v}}, acc, _), + do: update_in(acc, [:ps], fn ps -> Map.put(ps || %{}, k, v) end) + + defp handle_auth_pkts(%{tag: :ready_for_query, payload: db_state}, acc, _), + do: {:ready_for_query, Map.put(acc, :db_state, db_state)} + + defp handle_auth_pkts(%{tag: :backend_key_data, payload: payload}, acc, _), + do: Map.put(acc, :backend_key_data, payload) + + defp handle_auth_pkts(%{payload: {:authentication_sasl_password, methods_b}}, _, data) do + nonce = + case Server.decode_string(methods_b) do + {:ok, req_method, _} -> + Logger.debug("DbHandler: SASL method #{inspect(req_method)}") + nonce = :pgo_scram.get_nonce(16) + user = get_user(data.auth) + client_first = :pgo_scram.get_client_first(user, nonce) + client_first_size = IO.iodata_length(client_first) + + sasl_initial_response = [ + "SCRAM-SHA-256", + 0, + <>, + client_first + ] + + bin = :pgo_protocol.encode_scram_response_message(sasl_initial_response) + :ok = HandlerHelpers.sock_send(data.sock, bin) + nonce + + other -> + Logger.error("DbHandler: Undefined sasl method #{inspect(other)}") + nil + end + + {:authentication_sasl, nonce} + end + + defp handle_auth_pkts( + %{payload: {:authentication_server_first_message, server_first}}, + _, + data + ) + when data.auth.require_user == false do + nonce = data.nonce + server_first_parts = Helpers.parse_server_first(server_first, nonce) + + {client_final_message, server_proof} = + Helpers.get_client_final( + :auth_query, + data.auth.secrets.(), + server_first_parts, + nonce, + data.auth.secrets.().user, + "biws" + ) + + bin = :pgo_protocol.encode_scram_response_message(client_final_message) + :ok = HandlerHelpers.sock_send(data.sock, bin) + + {:authentication_server_first_message, server_proof} + end + + defp handle_auth_pkts( + %{payload: {:authentication_server_first_message, server_first}}, + _, + data + ) do + nonce = data.nonce + server_first_parts = :pgo_scram.parse_server_first(server_first, nonce) + + {client_final_message, server_proof} = + :pgo_scram.get_client_final( + server_first_parts, + nonce, + data.auth.user, + data.auth.secrets.().password + ) + + bin = :pgo_protocol.encode_scram_response_message(client_final_message) + :ok = HandlerHelpers.sock_send(data.sock, bin) + + {:authentication_server_first_message, server_proof} + end + + defp handle_auth_pkts( + %{payload: {:authentication_server_final_message, server_final}}, + acc, + _data + ), + do: Map.put(acc, :authentication_server_final_message, server_final) + + defp handle_auth_pkts( + %{payload: :authentication_ok}, + acc, + _data + ), + do: Map.put(acc, :authentication_ok, true) + + defp handle_auth_pkts(%{payload: {:authentication_md5_password, salt}} = dec_pkt, _, data) do + Logger.debug("DbHandler: dec_pkt, #{inspect(dec_pkt, pretty: true)}") + + digest = + if data.auth.method == :password do + Helpers.md5([data.auth.password.(), data.auth.user]) + else + data.auth.secrets.().secret + end + + payload = ["md5", Helpers.md5([digest, salt]), 0] + bin = [?p, <>, payload] + :ok = HandlerHelpers.sock_send(data.sock, bin) + :authentication_md5 + end + + defp handle_auth_pkts(%{tag: :error_response, payload: error}, _acc, _data), + do: {:error_response, error} + + defp handle_auth_pkts(_e, acc, _data), do: acc end diff --git a/lib/supavisor/handlers/proxy/client.ex b/lib/supavisor/handlers/proxy/client.ex deleted file mode 100644 index 7bed2727..00000000 --- a/lib/supavisor/handlers/proxy/client.ex +++ /dev/null @@ -1,787 +0,0 @@ -defmodule Supavisor.Handlers.Proxy.Client do - @moduledoc false - - require Logger - - alias Supavisor.{ - Tenants, - Helpers, - DbHandler, - HandlerHelpers, - Protocol.Server, - Monitoring.Telem, - Handlers.Proxy.Db - } - - @cancel_query_msg <<16::32, 1234::16, 5678::16>> - @sock_closed [:tcp_closed, :ssl_closed] - @proto [:tcp, :ssl] - - def handle_event(:info, {_proto, _, <<"GET", _::binary>>}, :exchange, data) do - Logger.debug("ProxyClient: Client is trying to request HTTP") - - HandlerHelpers.sock_send( - data.sock, - "HTTP/1.1 204 OK\r\nx-app-version: #{data.version}\r\n\r\n" - ) - - {:stop, {:shutdown, :http_request}} - end - - # cancel request - def handle_event(:info, {_, _, <<@cancel_query_msg, pid::32, key::32>>}, _, _) do - Logger.debug("ProxyClient: Got cancel query for #{inspect({pid, key})}") - :ok = HandlerHelpers.send_cancel_query(pid, key, {:client, :cancel_query}) - {:stop, {:shutdown, :cancel_query}} - end - - def handle_event(:info, {:client, :cancel_query}, _, %{db_pid: db_pid}) - when is_pid(db_pid) do - Logger.debug("ProxyClient: Cancel query for #{inspect(db_pid)}") - - case db_pid_meta(db_pid) do - [{^db_pid, meta}] -> - :ok = HandlerHelpers.cancel_query(meta.host, meta.port, meta.ip_ver, meta.pid, meta.key) - - error -> - Logger.error( - "ClientHandler: Received cancel but no proc was found #{inspect(db_pid)} #{inspect(error)}" - ) - end - - :keep_state_and_data - end - - def handle_event(:info, {:client, :cancel_query}, _, %{ - auth: auth, - backend_key_data: b - }) do - if b != %{} do - :ok = HandlerHelpers.cancel_query(~c"#{auth.host}", auth.port, auth.ip_ver, b.pid, b.key) - end - - :keep_state_and_data - end - - # # ssl request from client - def handle_event(:info, {:tcp, _, <<_::64>>}, :exchange, %{sock: sock} = data) do - Logger.debug("ProxyClient: Client is trying to connect with SSL") - - downstream_cert = Helpers.downstream_cert() - downstream_key = Helpers.downstream_key() - - # SSL negotiation, S/N/Error - if !!downstream_cert and !!downstream_key do - :ok = HandlerHelpers.setopts(sock, active: false) - :ok = HandlerHelpers.sock_send(sock, "S") - - opts = [ - certfile: downstream_cert, - keyfile: downstream_key - ] - - case :ssl.handshake(elem(sock, 1), opts) do - {:ok, ssl_sock} -> - socket = {:ssl, ssl_sock} - :ok = HandlerHelpers.setopts(socket, active: true) - {:keep_state, %{data | sock: socket, ssl: true}} - - error -> - Logger.error("ProxyClient: SSL handshake error: #{inspect(error)}") - Telem.client_join(:fail, data.id) - {:stop, {:shutdown, :ssl_handshake_error}} - end - else - Logger.error("ProxyClient: User requested SSL connection but no downstream cert/key found") - - :ok = HandlerHelpers.sock_send(data.sock, "N") - :keep_state_and_data - end - end - - def handle_event(:info, {_, _, bin}, :exchange, data) do - case Server.decode_startup_packet(bin) do - {:ok, hello} -> - Logger.debug("ProxyClient: Client startup message: #{inspect(hello)}") - {type, {user, tenant_or_alias, db_name}} = HandlerHelpers.parse_user_info(hello.payload) - - # Validate user and db_name according to PostgreSQL rules. - # The rules are: 1-63 characters, alphanumeric, underscore and $ - # TODO: spaces are allowed in db_name, but we don't support it yet - rule = ~r/^[a-z_][a-z0-9_$]*$/ - - if user =~ rule and db_name =~ rule do - log_level = maybe_change_log(hello) - event = {:hello, {type, {user, tenant_or_alias, db_name}}} - app_name = app_name(hello.payload["application_name"]) - - {:keep_state, %{data | log_level: log_level, app_name: app_name}, - {:next_event, :internal, {:client, event}}} - else - reason = "Invalid format for user or db_name" - Logger.error("ProxyClient: #{inspect(reason)}") - Telem.client_join(:fail, tenant_or_alias) - - HandlerHelpers.send_error( - data.sock, - "XX000", - "Authentication error, reason: #{inspect(reason)}" - ) - - {:stop, {:shutdown, :invalid_format}} - end - - {:error, error} -> - Logger.error("ProxyClient: Client startup message error: #{inspect(error)}") - Telem.client_join(:fail, data.id) - {:stop, {:shutdown, :startup_packet_error}} - end - end - - def handle_event( - :internal, - {:client, {:hello, {type, {user, tenant_or_alias, db_name}}}}, - :exchange, - %{sock: sock} = data - ) do - sni_hostname = HandlerHelpers.try_get_sni(sock) - - case Tenants.get_user_cache(type, user, tenant_or_alias, sni_hostname) do - {:ok, info} -> - db_name = if(db_name != nil, do: db_name, else: info.tenant.db_database) - - id = - Supavisor.id( - {type, tenant_or_alias}, - user, - data.mode, - info.user.mode_type, - db_name - ) - - mode = Supavisor.mode(id) - - Logger.metadata( - project: tenant_or_alias, - user: user, - mode: mode, - type: type, - db_name: db_name, - app_name: data.app_name, - peer_ip: data.peer_ip - ) - - Registry.register(Supavisor.Registry.TenantClients, id, []) - - {:ok, addr} = HandlerHelpers.addr_from_sock(sock) - - cond do - info.tenant.enforce_ssl and !data.ssl and !data.local -> - Logger.error( - "ProxyClient: Tenant is not allowed to connect without SSL, user #{user}" - ) - - :ok = HandlerHelpers.send_error(sock, "XX000", "SSL connection is required") - Telem.client_join(:fail, id) - {:stop, {:shutdown, :ssl_required}} - - HandlerHelpers.filter_cidrs(info.tenant.allow_list, addr) == [] -> - message = "Address not in tenant allow_list: " <> inspect(addr) - Logger.error("ProxyClient: #{message}") - :ok = HandlerHelpers.send_error(sock, "XX000", message) - - Telem.client_join(:fail, id) - {:stop, {:shutdown, :address_not_allowed}} - - true -> - new_data = update_user_data(data, info, user, id, db_name, mode) - - key = {:secrets, tenant_or_alias, user} - - case auth_secrets(info, user, key, :timer.hours(24)) do - {:ok, auth_secrets} -> - Logger.debug("ProxyClient: Authentication method: #{inspect(auth_secrets)}") - - event = {:handle, auth_secrets, info} - {:keep_state, new_data, {:next_event, :internal, {:client, event}}} - - {:error, reason} -> - Logger.error("ProxyClient: Authentication auth_secrets error: #{inspect(reason)}") - - :ok = - HandlerHelpers.send_error( - sock, - "XX000", - "Authentication error, reason: #{inspect(reason)}" - ) - - Telem.client_join(:fail, id) - {:stop, {:shutdown, :auth_secrets_error}} - end - end - - {:error, reason} -> - Logger.error( - "ProxyClient: User not found: #{inspect(reason)} #{inspect({type, user, tenant_or_alias})}" - ) - - :ok = HandlerHelpers.send_error(sock, "XX000", "Tenant or user not found") - Telem.client_join(:fail, data.id) - {:stop, {:shutdown, :user_not_found}} - end - end - - def handle_event( - :internal, - {:client, {:handle, {method, secrets}, info}}, - _, - %{sock: sock} = data - ) do - Logger.debug("ProxyClient: Handle exchange, auth method: #{inspect(method)}") - - case handle_exchange(sock, {method, secrets}) do - {:error, reason} -> - Logger.error( - "ProxyClient: Exchange error: #{inspect(reason)} when method #{inspect(method)}" - ) - - msg = - if method == :auth_query_md5, - do: Server.error_message("XX000", reason), - else: Server.exchange_message(:final, "e=#{reason}") - - key = {:secrets_check, data.tenant, data.user} - - if method != :password and reason == "Wrong password" and - Cachex.get(Supavisor.Cache, key) == {:ok, nil} do - case auth_secrets(info, data.user, key, 15_000) do - {:ok, {method2, secrets2}} = value -> - if method != method2 or Map.delete(secrets.(), :client_key) != secrets2.() do - Logger.warning("ProxyClient: Update secrets and terminate pool") - - Cachex.update( - Supavisor.Cache, - {:secrets, data.tenant, data.user}, - {:cached, value} - ) - - Supavisor.stop(data.id) - else - Logger.debug("ProxyClient: Cache the same #{inspect(key)}") - end - - other -> - Logger.error("ProxyClient: Auth secrets check error: #{inspect(other)}") - end - else - Logger.debug("ProxyClient: Cache hit for #{inspect(key)}") - end - - HandlerHelpers.sock_send(sock, msg) - Telem.client_join(:fail, data.id) - {:stop, {:shutdown, :exchange_error}} - - {:ok, client_key} -> - secrets = - if client_key do - fn -> - Map.put(secrets.(), :client_key, client_key) - end - else - secrets - end - - Logger.debug("ProxyClient: Exchange success") - :ok = HandlerHelpers.sock_send(sock, Server.authentication_ok()) - Telem.client_join(:ok, data.id) - - auth = Map.merge(data.auth, %{secrets: secrets, method: method}) - - conn_type = - case data.mode do - :transaction -> :subscribe - :session -> :connect_db - end - - {:keep_state, %{data | auth_secrets: {method, secrets}, auth: auth}, - {:next_event, :internal, {:client, conn_type}}} - end - end - - def handle_event(:internal, {:client, :subscribe}, _, data) do - Logger.warning("ClientHandler: Subscribe to tenant #{inspect(data.id)}") - - with {:ok, sup} <- - Supavisor.start_dist(data.id, data.auth_secrets, log_level: data.log_level), - true <- if(node(sup) == node(), do: true, else: :proxy), - {:ok, opts} <- Supavisor.subscribe(sup, data.id) do - Process.monitor(opts.workers.manager) - data = Map.merge(data, opts.workers) - data = %{data | idle_timeout: opts.idle_timeout} - - next = - if opts.ps == [] do - {:timeout, 10_000, :wait_ps} - else - {:next_event, :internal, {:client, {:greetings, opts.ps}}} - end - - {:keep_state, data, next} - else - {:error, :max_clients_reached} -> - msg = "Max client connections reached" - Logger.error("ClientHandler: #{msg}") - :ok = HandlerHelpers.send_error(data.sock, "XX000", msg) - Telem.client_join(:fail, data.id) - {:stop, {:shutdown, :max_clients_reached}} - - :proxy -> - {:ok, %{port: port, host: host}} = Supavisor.get_pool_ranch(data.id) - - auth = - Map.merge(data.auth, %{ - port: port, - host: host, - ip_version: :v4, - upstream_ssl: false, - upstream_tls_ca: nil, - upstream_verify: nil - }) - - data = Map.merge(data, %{auth: auth, mode: :session, proxy: true}) - {:keep_state, data, {:next_event, :internal, {:client, :connect_db}}} - - error -> - Logger.error("ClientHandler: Subscribe error: #{inspect(error)}") - {:keep_state_and_data, {:timeout, 1000, :subscribe}} - end - end - - def handle_event(:internal, {:client, :connect_db}, _, %{auth: auth} = data) do - Logger.debug("Try to connect to DB") - Telem.handler_action(:db_handler, :db_connection, data.id) - ip_ver = Helpers.ip_version(auth.ip_version, auth.host) - - sock_opts = [ - :binary, - {:packet, :raw}, - {:active, false}, - {:nodelay, true}, - ip_ver - ] - - case :gen_tcp.connect(~c"#{auth.host}", auth.port, sock_opts) do - {:ok, sock} -> - Logger.debug("ProxyClient: auth #{inspect(data, pretty: true)}") - - case Db.try_ssl_handshake({:gen_tcp, sock}, auth) do - {:ok, sock} -> - tenant = if(data.proxy, do: Supavisor.tenant(data.id)) - - case Db.send_startup(sock, auth, tenant) do - :ok -> - HandlerHelpers.active_once(sock) - auth = Map.put(auth, :ip_ver, ip_ver) - {:next_state, :db_authentication, %{data | db_sock: sock, auth: auth}} - - {:error, reason} -> - Logger.error("ProxyClient: Send startup error #{inspect(reason)}") - {:stop, {:shutdown, :startup_error}} - end - - {:error, error} -> - Logger.error("ProxyClient: Handshake error #{inspect(error)}") - {:stop, {:shutdown, :handshake_error}} - end - - other -> - Logger.error( - "ProxyClient: Connection failed #{inspect(other)} to #{inspect(auth.host)}:#{inspect(auth.port)}" - ) - - {:stop, {:shutdown, :connection_failed}} - end - end - - def handle_event(:internal, {:client, {:greetings, ps}}, _, %{sock: sock} = data) do - {header, <> = payload} = Server.backend_key_data() - msg = [ps, [header, payload], Server.ready_for_query()] - :ok = HandlerHelpers.listen_cancel_query(pid, key) - :ok = HandlerHelpers.sock_send(sock, msg) - HandlerHelpers.active_once(sock) - Telem.client_connection_time(data.connection_start, data.id) - {:next_state, :idle, data, handle_actions(data)} - end - - def handle_event(:timeout, :subscribe, _, _) do - {:keep_state_and_data, {:next_event, :internal, {:client, :connect_db}}} - end - - def handle_event(:timeout, :wait_ps, _, data) do - Logger.error("ProxyClient: Wait parameter status timeout, send default #{inspect(data.ps)}}") - - ps = Server.encode_parameter_status(data.ps) - {:keep_state_and_data, {:next_event, :internal, {:client, {:greetings, ps}}}} - end - - def handle_event(:timeout, :idle_terminate, _, data) do - Logger.warning("ProxyClient: Terminate an idle connection by #{data.idle_timeout} timeout") - {:stop, {:shutdown, :idle_terminate}} - end - - def handle_event(:timeout, :heartbeat_check, _, data) do - Logger.debug("ProxyClient: Send heartbeat to client") - HandlerHelpers.sock_send(data.sock, Server.application_name()) - {:keep_state_and_data, {:timeout, data.heartbeat_interval, :heartbeat_check}} - end - - def handle_event( - :info, - {proto, _, <>}, - _, - %{mode: :transaction, db_pid: nil} - ) - when proto in @proto do - {:stop, {:shutdown, :terminate_received}} - end - - # forwards the message to the db - def handle_event( - :info, - {proto, _, bin}, - _, - %{mode: :transaction, db_pid: nil} = data - ) - when proto in @proto do - {time, {db_pid, db_sock}} = :timer.tc(__MODULE__, :checkout, [data.pool, data.timeout]) - same_box = if node(db_pid) == node(), do: :local, else: :remote - Telem.pool_checkout_time(time, data.id, same_box) - Logger.debug("ProxyClient: Checkout new db connection #{inspect({db_pid, db_sock})}") - - HandlerHelpers.sock_send(db_sock, bin) - HandlerHelpers.active_once(data.sock) - {:keep_state, %{data | db_pid: db_pid, db_sock: db_sock}} - end - - def handle_event(:info, {proto, _, bin}, _, data) when proto in @proto do - HandlerHelpers.sock_send(data.db_sock, bin) - HandlerHelpers.active_once(data.sock) - :keep_state_and_data - end - - def handle_event(:info, {:parameter_status, ps}, :exchange, _), - do: {:keep_state_and_data, {:next_event, :internal, {:client, {:greetings, ps}}}} - - # client closed connection - def handle_event(_, {closed, _}, _, data) - when closed in @sock_closed do - Logger.debug("ProxyClient: #{closed} socket closed for #{inspect(data.tenant)}") - {:stop, {:shutdown, :socket_closed}} - end - - ## Internal functions - - @spec handle_exchange(Supavisor.sock(), {atom(), fun()}) :: - {:ok, binary() | nil} | {:error, String.t()} - def handle_exchange({_, socket} = sock, {:auth_query_md5 = method, secrets}) do - salt = :crypto.strong_rand_bytes(4) - :ok = HandlerHelpers.sock_send(sock, Server.md5_request(salt)) - - with {:ok, - %{ - tag: :password_message, - payload: {:md5, client_md5} - }, _} <- receive_next(socket, "Timeout while waiting for the md5 exchange"), - {:ok, key} <- authenticate_exchange(method, client_md5, secrets.().secret, salt) do - {:ok, key} - else - {:error, message} -> {:error, message} - other -> {:error, "Unexpected message #{inspect(other)}"} - end - end - - def handle_exchange({_, socket} = sock, {method, secrets}) do - :ok = HandlerHelpers.sock_send(sock, Server.scram_request()) - - with {:ok, - %{ - tag: :password_message, - payload: {:scram_sha_256, %{"n" => user, "r" => nonce, "c" => channel}} - }, - _} <- - receive_next( - socket, - "Timeout while waiting for the first password message" - ), - {:ok, signatures} = reply_first_exchange(sock, method, secrets, channel, nonce, user), - {:ok, - %{ - tag: :password_message, - payload: {:first_msg_response, %{"p" => p}} - }, - _} <- - receive_next( - socket, - "Timeout while waiting for the second password message" - ), - {:ok, key} <- authenticate_exchange(method, secrets, signatures, p) do - message = "v=#{Base.encode64(signatures.server)}" - :ok = HandlerHelpers.sock_send(sock, Server.exchange_message(:final, message)) - {:ok, key} - else - {:error, message} -> {:error, message} - other -> {:error, "Unexpected message #{inspect(other)}"} - end - end - - def receive_next(socket, timeout_message) do - receive do - {_proto, ^socket, bin} -> Server.decode_pkt(bin) - other -> {:error, "Unexpected message in receive_next/2 #{inspect(other)}"} - after - 15_000 -> {:error, timeout_message} - end - end - - def reply_first_exchange(sock, method, secrets, channel, nonce, user) do - {message, signatures} = exchange_first(method, secrets, nonce, user, channel) - :ok = HandlerHelpers.sock_send(sock, Server.exchange_message(:first, message)) - {:ok, signatures} - end - - def authenticate_exchange(:password, _secrets, signatures, p) do - if p == signatures.client, - do: {:ok, nil}, - else: {:error, "Wrong password"} - end - - def authenticate_exchange(:auth_query, secrets, signatures, p) do - client_key = :crypto.exor(Base.decode64!(p), signatures.client) - - if Helpers.hash(client_key) == secrets.().stored_key, - do: {:ok, client_key}, - else: {:error, "Wrong password"} - end - - def authenticate_exchange(:auth_query_md5, client_hash, server_hash, salt) do - if "md5" <> Helpers.md5([server_hash, salt]) == client_hash, - do: {:ok, nil}, - else: {:error, "Wrong password"} - end - - @spec update_user_data(map(), map(), String.t(), Supavisor.id(), String.t(), Supavisor.mode()) :: - map() - def update_user_data(data, info, user, id, db_name, mode) do - proxy_type = if info.tenant.require_user, do: :password, else: :auth_query - - auth = %{ - application_name: data.app_name, - database: info.tenant.db_database, - host: info.tenant.db_host, - sni_host: info.tenant.sni_hostname, - ip_version: info.tenant.ip_version, - port: info.tenant.db_port, - user: user, - password: info.user.db_password, - require_user: info.tenant.require_user, - upstream_ssl: info.tenant.upstream_ssl, - upstream_tls_ca: info.tenant.upstream_tls_ca, - upstream_verify: info.tenant.upstream_verify - } - - %{ - data - | tenant: info.tenant.external_id, - user: user, - timeout: info.user.pool_checkout_timeout, - ps: info.tenant.default_parameter_status, - proxy_type: proxy_type, - id: id, - heartbeat_interval: info.tenant.client_heartbeat_interval * 1000, - db_name: db_name, - mode: mode, - auth: auth - } - end - - @spec auth_secrets(map, String.t(), term(), non_neg_integer()) :: - {:ok, Supavisor.secrets()} | {:error, term()} - ## password secrets - def auth_secrets(%{user: user, tenant: %{require_user: true}}, _, _, _) do - secrets = %{db_user: user.db_user, password: user.db_password, alias: user.db_user_alias} - {:ok, {:password, fn -> secrets end}} - end - - ## auth_query secrets - def auth_secrets(info, db_user, key, ttl) do - fetch = fn _key -> - case get_secrets(info, db_user) do - {:ok, _} = resp -> {:commit, {:cached, resp}, ttl: ttl} - {:error, _} = resp -> {:ignore, resp} - end - end - - case Cachex.fetch(Supavisor.Cache, key, fetch) do - {:ok, {:cached, value}} -> value - {:commit, {:cached, value}, _opts} -> value - {:ignore, resp} -> resp - end - end - - @spec get_secrets(map, String.t()) :: {:ok, {:auth_query, fun()}} | {:error, term()} - def get_secrets(%{user: user, tenant: tenant}, db_user) do - ssl_opts = - if tenant.upstream_ssl and tenant.upstream_verify == :peer do - [ - {:verify, :verify_peer}, - {:cacerts, [Helpers.upstream_cert(tenant.upstream_tls_ca)]}, - {:server_name_indication, String.to_charlist(tenant.db_host)}, - {:customize_hostname_check, [{:match_fun, fn _, _ -> true end}]} - ] - end - - {:ok, conn} = - Postgrex.start_link( - hostname: tenant.db_host, - port: tenant.db_port, - database: tenant.db_database, - password: user.db_password, - username: user.db_user, - parameters: [application_name: "Supavisor auth_query"], - ssl: tenant.upstream_ssl, - socket_options: [ - Helpers.ip_version(tenant.ip_version, tenant.db_host) - ], - queue_target: 1_000, - queue_interval: 5_000, - ssl_opts: ssl_opts || [] - ) - - # kill the postgrex connection if the current process exits unexpectedly - Process.link(conn) - - Logger.debug( - "ProxyClient: Connected to db #{tenant.db_host} #{tenant.db_port} #{tenant.db_database} #{user.db_user}" - ) - - resp = - with {:ok, secret} <- Helpers.get_user_secret(conn, tenant.auth_query, db_user) do - t = if secret.digest == :md5, do: :auth_query_md5, else: :auth_query - {:ok, {t, fn -> Map.put(secret, :alias, user.db_user_alias) end}} - end - - GenServer.stop(conn, :normal, 5_000) - Logger.info("ProxyClient: Get secrets finished") - resp - end - - @spec exchange_first(:password | :auth_query, fun(), binary(), binary(), binary()) :: - {binary(), map()} - def exchange_first(:password, secret, nonce, user, channel) do - message = Server.exchange_first_message(nonce) - server_first_parts = Helpers.parse_server_first(message, nonce) - - {client_final_message, server_proof} = - Helpers.get_client_final( - :password, - secret.().password, - server_first_parts, - nonce, - user, - channel - ) - - sings = %{ - client: List.last(client_final_message), - server: server_proof - } - - {message, sings} - end - - def exchange_first(:auth_query, secret, nonce, user, channel) do - secret = secret.() - message = Server.exchange_first_message(nonce, secret.salt) - server_first_parts = Helpers.parse_server_first(message, nonce) - - sings = - Helpers.signatures( - secret.stored_key, - secret.server_key, - server_first_parts, - nonce, - user, - channel - ) - - {message, sings} - end - - @spec timeout_check(atom, non_neg_integer) :: {:timeout, non_neg_integer, atom} - def timeout_check(key, timeout) do - {:timeout, timeout, key} - end - - @spec handle_actions(map) :: [{:timeout, non_neg_integer, atom}] - defp handle_actions(%{} = data) do - heartbeat = - if data.heartbeat_interval > 0, - do: [{:timeout, data.heartbeat_interval, :heartbeat_check}], - else: [] - - idle = - if data.idle_timeout > 0, do: [{:timeout, data.idle_timeout, :idle_timeout}], else: [] - - idle ++ heartbeat - end - - @spec app_name(any()) :: String.t() - def app_name(name) when is_binary(name) do - suffix = " via Supavisor" - # https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-APPLICATION-NAME - max_len = 64 - suffix_len = 14 - - if byte_size(name) <= max_len - suffix_len do - name <> suffix - else - truncated_name = binary_slice(name, 0, max_len - suffix_len - 3) - truncated_name <> "..." <> suffix - end - end - - def app_name(name) do - Logger.error("ProxyClient: Invalid application name #{inspect(name)}") - "via Supavisor" - end - - @spec maybe_change_log(map()) :: atom() | nil - def maybe_change_log(%{"payload" => %{"options" => options}}) do - level = options["log_level"] && String.to_existing_atom(options["log_level"]) - - if level in [:debug, :info, :notice, :warning, :error] do - Helpers.set_log_level(level) - level - end - end - - def maybe_change_log(_), do: :ok - - @spec checkout(pid(), non_neg_integer()) :: {pid(), Supavisor.sock()} - def checkout(pool, timeout) do - db_pid = :poolboy.checkout(pool, true, timeout) - Process.link(db_pid) - {:ok, db_sock} = DbHandler.change_socket_owner(db_pid, self()) - {db_pid, db_sock} - end - - @spec db_pid_meta(pid()) :: [{pid(), map()}] - defp db_pid_meta(pid) do - rkey = Supavisor.Registry.PoolPids - fnode = node(pid) - - if fnode == node(), - do: Registry.lookup(rkey, pid), - else: :erpc.call(fnode, Registry, :lookup, [rkey, pid], 15_000) - end -end diff --git a/lib/supavisor/handlers/proxy/db.ex b/lib/supavisor/handlers/proxy/db.ex deleted file mode 100644 index ce6b604c..00000000 --- a/lib/supavisor/handlers/proxy/db.ex +++ /dev/null @@ -1,298 +0,0 @@ -defmodule Supavisor.Handlers.Proxy.Db do - @moduledoc false - - require Logger - - alias Supavisor.{ - Helpers, - DbHandler, - HandlerHelpers, - Monitoring.Telem, - Protocol.Server - } - - @type state :: :connect | :authentication | :idle | :busy - - @sock_closed [:tcp_closed, :ssl_closed] - @proto [:tcp, :ssl] - - def handle_event(:info, {proto, _, bin}, :db_authentication, data) when proto in @proto do - dec_pkt = Server.decode(bin) - Logger.debug("ProxyDb: dec_pkt, #{inspect(dec_pkt, pretty: true)}") - HandlerHelpers.active_once(data.db_sock) - - resp = Enum.reduce(dec_pkt, %{}, &handle_auth_pkts(&1, &2, data)) - - case resp do - {:authentication_sasl, nonce} -> - {:keep_state, %{data | nonce: nonce}} - - {:authentication_server_first_message, server_proof} -> - {:keep_state, %{data | server_proof: server_proof}} - - %{authentication_server_final_message: _server_final} -> - :keep_state_and_data - - %{authentication_ok: true} -> - :keep_state_and_data - - :authentication -> - :keep_state_and_data - - :authentication_md5 -> - {:keep_state, data} - - {:error_response, ["SFATAL", "VFATAL", "C28P01", reason, _, _, _]} -> - Logger.error("ProxyDb: Auth error #{inspect(reason)}") - {:stop, :invalid_password, data} - - {:error_response, error} -> - Logger.error("ProxyDb: Error auth response #{inspect(error)}") - {:keep_state, data} - - {:ready_for_query, acc} -> - ps = acc.ps - backend_key_data = acc.backend_key_data - msg = "ProxyDb: DB ready_for_query: #{inspect(acc.db_state)} #{inspect(ps, pretty: true)}" - Logger.debug(msg) - ps_encoded = Server.encode_parameter_status(ps) - - {:next_state, :idle, %{data | parameter_status: ps, backend_key_data: backend_key_data}, - {:next_event, :internal, {:client, {:greetings, ps_encoded}}}} - - other -> - Logger.error("ProxyDb: Undefined auth response #{inspect(other)}") - {:stop, :auth_error, data} - end - end - - # forwards the message to the client - def handle_event(:info, {proto, _, bin}, _, data) when proto in @proto do - HandlerHelpers.sock_send(data.sock, bin) - HandlerHelpers.active_once(data.db_sock) - - data = - if String.ends_with?(bin, Server.ready_for_query()) do - Logger.debug("ProxyDb: collected network usage") - {_, stats} = Telem.network_usage(:client, data.sock, data.id, data.stats) - {_, db_stats} = Telem.network_usage(:db, data.db_sock, data.id, data.db_stats) - - case data.mode do - :transaction -> - DbHandler.set_idle(data.db_pid) - Helpers.controlling_process(data.db_sock, data.db_pid) - Process.unlink(data.db_pid) - :poolboy.checkin(data.pool, data.db_pid) - %{data | stats: stats, db_stats: db_stats, db_pid: nil, db_sock: nil} - - _ -> - %{data | stats: stats, db_stats: db_stats} - end - else - data - end - - {:keep_state, data} - end - - def handle_event(_, {closed, _}, state, data) when closed in @sock_closed do - Logger.error("ProxyDb: Connection closed when state was #{state}") - Telem.handler_action(:db_handler, :stopped, data.id) - - HandlerHelpers.sock_send( - data.sock, - Server.error_message("XX000", "Database connection closed") - ) - - HandlerHelpers.sock_close(data.sock) - {:stop, :db_socket_closed, data} - end - - ## Internal functions - - @spec handle_auth_pkts(map(), map(), map()) :: any() - defp handle_auth_pkts(%{tag: :parameter_status, payload: {k, v}}, acc, _), - do: update_in(acc, [:ps], fn ps -> Map.put(ps || %{}, k, v) end) - - defp handle_auth_pkts(%{tag: :ready_for_query, payload: db_state}, acc, _), - do: {:ready_for_query, Map.put(acc, :db_state, db_state)} - - defp handle_auth_pkts(%{tag: :backend_key_data, payload: payload}, acc, _), - do: Map.put(acc, :backend_key_data, payload) - - defp handle_auth_pkts(%{payload: {:authentication_sasl_password, methods_b}}, _, data) do - nonce = - case Server.decode_string(methods_b) do - {:ok, req_method, _} -> - Logger.debug("ProxyDb: SASL method #{inspect(req_method)}") - nonce = :pgo_scram.get_nonce(16) - user = get_user(data.auth) - client_first = :pgo_scram.get_client_first(user, nonce) - client_first_size = IO.iodata_length(client_first) - - sasl_initial_response = [ - "SCRAM-SHA-256", - 0, - <>, - client_first - ] - - bin = :pgo_protocol.encode_scram_response_message(sasl_initial_response) - :ok = HandlerHelpers.sock_send(data.db_sock, bin) - nonce - - other -> - Logger.error("ProxyDb: Undefined sasl method #{inspect(other)}") - nil - end - - {:authentication_sasl, nonce} - end - - defp handle_auth_pkts( - %{payload: {:authentication_server_first_message, server_first}}, - _, - data - ) - when data.auth.require_user == false do - nonce = data.nonce - server_first_parts = Helpers.parse_server_first(server_first, nonce) - - {client_final_message, server_proof} = - Helpers.get_client_final( - :auth_query, - data.auth.secrets.(), - server_first_parts, - nonce, - data.auth.secrets.().user, - "biws" - ) - - bin = :pgo_protocol.encode_scram_response_message(client_final_message) - :ok = HandlerHelpers.sock_send(data.db_sock, bin) - - {:authentication_server_first_message, server_proof} - end - - defp handle_auth_pkts( - %{payload: {:authentication_server_first_message, server_first}}, - _, - data - ) do - nonce = data.nonce - server_first_parts = :pgo_scram.parse_server_first(server_first, nonce) - - {client_final_message, server_proof} = - :pgo_scram.get_client_final( - server_first_parts, - nonce, - data.auth.user, - data.auth.secrets.().password - ) - - bin = :pgo_protocol.encode_scram_response_message(client_final_message) - :ok = HandlerHelpers.sock_send(data.db_sock, bin) - - {:authentication_server_first_message, server_proof} - end - - defp handle_auth_pkts( - %{payload: {:authentication_server_final_message, server_final}}, - acc, - _data - ), - do: Map.put(acc, :authentication_server_final_message, server_final) - - defp handle_auth_pkts( - %{payload: :authentication_ok}, - acc, - _data - ), - do: Map.put(acc, :authentication_ok, true) - - defp handle_auth_pkts(%{payload: {:authentication_md5_password, salt}} = dec_pkt, _, data) do - Logger.debug("ProxyDb: dec_pkt, #{inspect(dec_pkt, pretty: true)}") - - digest = - if data.auth.method == :password do - Helpers.md5([data.auth.password.(), data.auth.user]) - else - data.auth.secrets.().secret - end - - payload = ["md5", Helpers.md5([digest, salt]), 0] - bin = [?p, <>, payload] - :ok = HandlerHelpers.sock_send(data.db_sock, bin) - :authentication_md5 - end - - defp handle_auth_pkts(%{tag: :error_response, payload: error}, _acc, _data), - do: {:error_response, error} - - defp handle_auth_pkts(_e, acc, _data), do: acc - - @spec try_ssl_handshake(Supavisor.tcp_sock(), map()) :: - {:ok, Supavisor.sock()} | {:error, term()} - def try_ssl_handshake(sock, %{upstream_ssl: true} = auth) do - with :ok <- HandlerHelpers.sock_send(sock, Server.ssl_request()) do - ssl_recv(sock, auth) - end - end - - def try_ssl_handshake(sock, _), do: {:ok, sock} - - @spec ssl_recv(Supavisor.tcp_sock(), map) :: {:ok, Supavisor.ssl_sock()} | {:error, term} - def ssl_recv({:gen_tcp, sock} = s, auth) do - case :gen_tcp.recv(sock, 1, 15_000) do - {:ok, <>} -> ssl_connect(s, auth) - {:ok, <>} -> {:error, :ssl_not_available} - {:error, _} = error -> error - end - end - - @spec ssl_connect(Supavisor.tcp_sock(), map, pos_integer) :: - {:ok, Supavisor.ssl_sock()} | {:error, term} - def ssl_connect({:gen_tcp, sock}, auth, timeout \\ 5000) do - opts = - case auth.upstream_verify do - :peer -> - [ - verify: :verify_peer, - cacerts: [auth.upstream_tls_ca], - # unclear behavior on pg14 - server_name_indication: auth.sni_host || auth.host, - customize_hostname_check: [{:match_fun, fn _, _ -> true end}] - ] - - :none -> - [verify: :verify_none] - end - - case :ssl.connect(sock, opts, timeout) do - {:ok, ssl_sock} -> {:ok, {:ssl, ssl_sock}} - {:error, reason} -> {:error, reason} - end - end - - @spec send_startup(Supavisor.sock(), map(), String.t() | nil) :: :ok | {:error, term} - def send_startup(sock, auth, tenant) do - user = - if is_nil(tenant), do: get_user(auth), else: "#{get_user(auth)}.#{tenant}" - - msg = - :pgo_protocol.encode_startup_message([ - {"user", user}, - {"database", auth.database}, - {"application_name", auth.application_name} - ]) - - HandlerHelpers.sock_send(sock, msg) - end - - @spec get_user(map) :: String.t() - def get_user(auth) do - if auth.require_user, - do: auth.secrets.().db_user, - else: auth.secrets.().user - end -end diff --git a/lib/supavisor/handlers/proxy/handler.ex b/lib/supavisor/handlers/proxy/handler.ex deleted file mode 100644 index c08045d5..00000000 --- a/lib/supavisor/handlers/proxy/handler.ex +++ /dev/null @@ -1,156 +0,0 @@ -defmodule Supavisor.Handlers.Proxy.Handler do - @moduledoc false - - require Logger - - @behaviour :ranch_protocol - @behaviour :gen_statem - - alias Supavisor.{ - Helpers, - HandlerHelpers, - Protocol.Server, - Monitoring.PromEx, - Handlers.Proxy.Db, - Handlers.Proxy.Client - } - - @metrics_disabled Application.compile_env(:supavisor, :metrics_disabled, false) - - @sock_closed [:tcp_closed, :ssl_closed] - @proto [:tcp, :ssl] - - @impl true - def start_link(ref, transport, opts) do - pid = :proc_lib.spawn_link(__MODULE__, :init, [ref, transport, opts]) - {:ok, pid} - end - - @impl true - def callback_mode, do: [:handle_event_function] - - @impl true - def init(_), do: :ignore - - def init(ref, trans, opts) do - Process.flag(:trap_exit, true) - Helpers.set_max_heap_size(90) - - {:ok, sock} = :ranch.handshake(ref) - :ok = trans.setopts(sock, active: true) - Logger.debug("ProxyHandler is: #{inspect(self())}") - - data = %{ - id: nil, - sock: {:gen_tcp, sock}, - db_sock: {:gen_tcp, nil}, - trans: trans, - db_pid: nil, - tenant: nil, - user: nil, - pool: nil, - manager: nil, - query_start: nil, - timeout: nil, - ps: nil, - ssl: false, - auth_secrets: nil, - proxy_type: nil, - mode: opts.mode, - stats: %{}, - db_stats: %{}, - idle_timeout: 0, - db_name: nil, - last_query: nil, - heartbeat_interval: 0, - connection_start: System.monotonic_time(), - log_level: nil, - version: Application.spec(:supavisor, :vsn), - nonce: nil, - server_proof: nil, - parameter_status: %{}, - app_name: nil, - peer_ip: Helpers.peer_ip(sock), - auth: %{}, - backend_key_data: %{}, - local: opts[:local] || false, - proxy: false - } - - :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) - end - - @impl true - def handle_event(:info, {:parameter_status, ps}, :exchange, _), - do: {:keep_state_and_data, {:next_event, :internal, {:client, {:greetings, ps}}}} - - def handle_event(e, {:client, _} = msg, state, data), - do: Client.handle_event(e, msg, state, data) - - def handle_event(:timeout = e, msg, state, data), - do: Client.handle_event(e, msg, state, data) - - def handle_event(event, {proto, sock, _payload} = msg, state, %{sock: {_, sock}} = data) - when proto in @proto, - do: Client.handle_event(event, msg, state, data) - - def handle_event(event, {proto, sock, _payload} = msg, state, %{db_sock: {_, sock}} = data) - when proto in @proto, - do: Db.handle_event(event, msg, state, data) - - def handle_event(event, {closed, sock} = msg, state, %{sock: {_, sock}} = data) - when closed in @sock_closed, - do: Client.handle_event(event, msg, state, data) - - def handle_event(event, {closed, sock} = msg, state, %{db_sock: {_, sock}} = data) - when closed in @sock_closed, - do: Db.handle_event(event, msg, state, data) - - def handle_event(type, content, state, data) do - msg = [ - {"type", type}, - {"content", content}, - {"state", state}, - {"data", data} - ] - - Logger.debug("ProxyHandler: Undefined msg: #{inspect(msg, pretty: true)}") - - :keep_state_and_data - end - - @impl true - def terminate({:shutdown, reason}, state, data) do - HandlerHelpers.sock_send(data.sock, Server.error_message("XX000", "#{inspect(reason)}")) - clean_up(data) - - Logger.info( - "ProxyHandler: Terminating with reason: #{inspect(reason)} when state was #{state}" - ) - - :ok - end - - def terminate(reason, state, data) do - clean_up(data) - - Logger.info( - "ProxyHandler: Terminating with reason: #{inspect(reason)} when state was #{state}" - ) - end - - ## Internal functions - - @spec clean_up(map()) :: any() - defp clean_up(data) do - HandlerHelpers.sock_close(data.sock) - HandlerHelpers.sock_close(data.db_sock) - - if not @metrics_disabled and data.id != nil do - case Registry.lookup(Supavisor.Registry.TenantClients, data.id) do - clients when clients == [{self(), []}] or clients == [] -> PromEx.remove_metrics(data.id) - _ -> :ok - end - end - end -end diff --git a/test/supavisor/db_handler_test.exs b/test/supavisor/db_handler_test.exs index efccc86a..f61ec753 100644 --- a/test/supavisor/db_handler_test.exs +++ b/test/supavisor/db_handler_test.exs @@ -57,7 +57,8 @@ defmodule Supavisor.DbHandlerTest do Db.handle_event(:internal, nil, :connect, %{ auth: auth, sock: {:gen_tcp, nil}, - id: {"a", "b"} + id: {"a", "b"}, + proxy: false }) assert state == @@ -74,7 +75,8 @@ defmodule Supavisor.DbHandlerTest do secrets: secrets }, sock: {:gen_tcp, :sock}, - id: {"a", "b"} + id: {"a", "b"}, + proxy: false }} :meck.unload(:gen_tcp) @@ -102,34 +104,6 @@ defmodule Supavisor.DbHandlerTest do end end - test "handle_event/4 with idle state" do - {:ok, sock} = :gen_tcp.listen(0, []) - data = %{sock: {:gen_tcp, sock}, caller: nil, buffer: []} - from = {self(), :test_ref} - event = {:call, from} - payload = {:db_call, self(), "test_data"} - - {:next_state, :busy, new_data, reply} = Db.handle_event(event, payload, :idle, data) - - # check if the message arrived in gen_tcp.send - assert {:reply, ^from, {:error, :enotconn}} = reply - assert new_data.caller == self() - end - - test "handle_event/4 with non-idle state" do - data = %{sock: nil, caller: self(), buffer: []} - from = {self(), :test_ref} - event = {:call, from} - payload = {:db_call, self(), "test_data"} - state = :non_idle - - {:keep_state, new_data, reply} = Db.handle_event(event, payload, state, data) - - assert {:reply, ^from, {:buffering, 9}} = reply - assert new_data.caller == self() - assert new_data.buffer == ["test_data"] - end - describe "handle_event/4 info tcp authentication authentication_md5_password payload events" do test "keeps state while sending the digested md5" do # `82` is `?R`, which identifies the payload tag as `:authentication` @@ -197,41 +171,8 @@ defmodule Supavisor.DbHandlerTest do :meck.expect(:inet, :setopts, fn _, _ -> :ok end) - {:next_state, :idle, new_data, _} = Db.handle_event(:info, event, state, data) - - assert new_data.caller == caller_pid - :meck.unload(:prim_inet) - :meck.unload(:inet) - end - - test "does not update caller in data for non-session mode" do - proto = :tcp - bin = "response_data" <> Server.ready_for_query() - caller_pid = self() - - data = %{ - id: {{:single, "tenant"}, "user", :session, "postgres"}, - caller: caller_pid, - sock: {:gen_tcp, nil}, - stats: %{}, - mode: :transaction, - sent: false - } - - state = :some_state - event = {proto, :dummy_value, bin} - :meck.new(:prim_inet, [:unstick, :passthrough]) - :meck.new(:inet, [:unstick, :passthrough]) - - :meck.expect(:prim_inet, :getstat, fn _, _ -> - {:ok, [{:recv_oct, 21}, {:send_oct, 37}]} - end) - - :meck.expect(:inet, :setopts, fn _, _ -> :ok end) - - {:next_state, :idle, new_data, _} = Db.handle_event(:info, event, state, data) + :keep_state_and_data = Db.handle_event(:info, event, state, data) - assert new_data.caller == nil :meck.unload(:prim_inet) :meck.unload(:inet) end From 79407613146436958641ff2dbceb7fd306825c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Wed, 21 Aug 2024 20:00:36 +0200 Subject: [PATCH 38/97] style: clean Credo warnings (#413) --- .github/workflows/elixir.yml | 2 +- lib/cluster/strategy/postgres.ex | 2 +- lib/supavisor.ex | 10 +++++++- lib/supavisor/application.ex | 9 ++++--- lib/supavisor/client_handler.ex | 6 ++--- lib/supavisor/db_handler.ex | 18 ++++++++------ lib/supavisor/helpers.ex | 24 +++++++++---------- lib/supavisor/hot_upgrade.ex | 18 +++++++------- lib/supavisor/manager.ex | 6 ++--- lib/supavisor/monitoring/osmon.ex | 8 +++---- lib/supavisor/monitoring/prom_ex.ex | 8 +++---- lib/supavisor/monitoring/telem.ex | 6 ++--- lib/supavisor/monitoring/tenant.ex | 8 +++---- lib/supavisor/native_handler.ex | 19 ++++++++------- lib/supavisor/protocol/client.ex | 2 +- lib/supavisor/protocol/server.ex | 20 ++++++++-------- lib/supavisor/tenants.ex | 4 ++-- lib/supavisor/tenants/cluster_tenants.ex | 4 +++- lib/supavisor/tenants_metrics.ex | 2 +- lib/supavisor_web/api_spec.ex | 2 +- .../controllers/metrics_controller.ex | 2 +- .../controllers/tenant_controller.ex | 15 ++++++++---- lib/supavisor_web/open_api_schemas.ex | 12 +++++----- lib/supavisor_web/views/cluster_view.ex | 3 ++- lib/supavisor_web/ws_proxy.ex | 2 +- 25 files changed, 117 insertions(+), 95 deletions(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index bede91b5..95ca8314 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -80,7 +80,7 @@ jobs: - name: Compile deps run: mix deps.compile - name: Credo checks - run: mix credo --strict --mute-exit-status + run: mix credo --strict --all --mute-exit-status tests: name: Run tests diff --git a/lib/cluster/strategy/postgres.ex b/lib/cluster/strategy/postgres.ex index 2020aacb..08d8484e 100644 --- a/lib/cluster/strategy/postgres.ex +++ b/lib/cluster/strategy/postgres.ex @@ -18,8 +18,8 @@ defmodule Cluster.Strategy.Postgres do @vsn "1.1.49" - alias Cluster.Strategy alias Cluster.Logger + alias Cluster.Strategy alias Postgrex, as: P def start_link(args), do: GenServer.start_link(__MODULE__, args) diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 6a3c2590..8f6f1be1 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -1,7 +1,15 @@ defmodule Supavisor do @moduledoc false + require Logger - alias Supavisor.{Manager, Helpers, Tenants} + + import Cachex.Spec + + alias Supavisor.{ + Helpers, + Manager, + Tenants + } @type sock :: tcp_sock() | ssl_sock() @type ssl_sock :: {:ssl, :ssl.sslsocket()} diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index a5ae9ac8..ca0fbdd8 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -4,7 +4,10 @@ defmodule Supavisor.Application do @moduledoc false use Application + require Logger + + alias Supavisor.Handlers.Proxy.Handler, as: ProxyHandler alias Supavisor.Monitoring.PromEx @metrics_disabled Application.compile_env(:supavisor, :metrics_disabled, false) @@ -88,11 +91,11 @@ defmodule Supavisor.Application do Logger.warning("metrics_disabled is #{inspect(@metrics_disabled)}") children = - if not @metrics_disabled do + if @metrics_disabled do + children + else PromEx.set_metrics_tags() children ++ [PromEx, Supavisor.TenantsMetrics] - else - children end # start Cachex only if the node uses names, this is necessary for test setup diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index e24d6f99..3324dd53 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -14,13 +14,13 @@ defmodule Supavisor.ClientHandler do @cancel_query_msg <<16::32, 1234::16, 5678::16>> alias Supavisor.{ - Tenants, - Helpers, DbHandler, HandlerHelpers, + Helpers, + Monitoring.Telem, Protocol.Client, Protocol.Server, - Monitoring.Telem + Tenants } @impl true diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index ed7f71ad..664a5686 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -8,7 +8,13 @@ defmodule Supavisor.DbHandler do @behaviour :gen_statem - alias Supavisor.{Helpers, HandlerHelpers, ClientHandler, Monitoring.Telem, Protocol.Server} + alias Supavisor.{ + ClientHandler, + HandlerHelpers, + Helpers, + Monitoring.Telem, + Protocol.Server + } @type state :: :connect | :authentication | :idle | :busy @@ -27,11 +33,9 @@ defmodule Supavisor.DbHandler do @spec get_state_and_mode(pid()) :: {:ok, {state, Supavisor.mode()}} | {:error, term()} def get_state_and_mode(pid) do - try do - {:ok, :gen_statem.call(pid, :get_state_and_mode, 5_000)} - catch - error, reason -> {:error, {error, reason}} - end + {:ok, :gen_statem.call(pid, :get_state_and_mode, 5_000)} + catch + error, reason -> {:error, {error, reason}} end @spec stop(pid()) :: :ok @@ -459,7 +463,7 @@ defmodule Supavisor.DbHandler do end @spec receive_ready_for_query() :: :ok | :timeout_error - defp receive_ready_for_query() do + defp receive_ready_for_query do receive do {_proto, _socket, <>} -> :ok diff --git a/lib/supavisor/helpers.ex b/lib/supavisor/helpers.ex index d0c8062f..0177836b 100644 --- a/lib/supavisor/helpers.ex +++ b/lib/supavisor/helpers.ex @@ -57,7 +57,9 @@ defmodule Supavisor.Helpers do Postgrex.query(conn, "select version()", []) |> case do {:ok, %{rows: [[version]]}} -> - if !params["require_user"] do + if params["require_user"] do + {:cont, {:ok, version}} + else case get_user_secret(conn, params["auth_query"], user["db_user"]) do {:ok, _} -> {:halt, {:ok, version}} @@ -65,8 +67,6 @@ defmodule Supavisor.Helpers do {:error, reason} -> {:halt, {:error, reason}} end - else - {:cont, {:ok, version}} end {:error, reason} -> @@ -231,12 +231,12 @@ defmodule Supavisor.Helpers do end @spec downstream_cert() :: Path.t() | nil - def downstream_cert() do + def downstream_cert do Application.get_env(:supavisor, :global_downstream_cert) end @spec downstream_key() :: Path.t() | nil - def downstream_key() do + def downstream_key do Application.get_env(:supavisor, :global_downstream_key) end @@ -327,14 +327,12 @@ defmodule Supavisor.Helpers do @spec rpc(Node.t(), module(), atom(), [any()], non_neg_integer()) :: {:error, any()} | any() def rpc(node, module, function, args, timeout \\ 15_000) do - try do - :erpc.call(node, module, function, args, timeout) - catch - kind, reason -> {:error, {:badrpc, {kind, reason}}} - else - {:EXIT, _} = badrpc -> {:error, {:badrpc, badrpc}} - result -> result - end + :erpc.call(node, module, function, args, timeout) + catch + kind, reason -> {:error, {:badrpc, {kind, reason}}} + else + {:EXIT, _} = badrpc -> {:error, {:badrpc, badrpc}} + result -> result end @doc """ diff --git a/lib/supavisor/hot_upgrade.ex b/lib/supavisor/hot_upgrade.ex index 682f56c5..7e2f296f 100644 --- a/lib/supavisor/hot_upgrade.ex +++ b/lib/supavisor/hot_upgrade.ex @@ -91,12 +91,12 @@ defmodule Supavisor.HotUpgrade do end end - def reint_funs() do + def reint_funs do reinit_pool_args() reinit_auth_query() end - def reinit_pool_args() do + def reinit_pool_args do for [_tenant, pid, _meta] <- Registry.select(Supavisor.Registry.TenantSups, [ {{:"$1", :"$2", :"$3"}, [], [[:"$1", :"$2", :"$3"]]} @@ -131,7 +131,7 @@ defmodule Supavisor.HotUpgrade do end end - def reinit_auth_query() do + def reinit_auth_query do Supavisor.Cache |> Cachex.stream!() |> Enum.each(fn entry(key: key, value: value) -> @@ -159,12 +159,10 @@ defmodule Supavisor.HotUpgrade do def do_enc(val), do: fn -> val end def get_state(pid) do - try do - {:ok, :sys.get_state(pid)} - catch - type, exception -> - IO.write("Error getting state: #{inspect(exception)}") - {:error, {type, exception}} - end + {:ok, :sys.get_state(pid)} + catch + type, exception -> + IO.write("Error getting state: #{inspect(exception)}") + {:error, {type, exception}} end end diff --git a/lib/supavisor/manager.ex b/lib/supavisor/manager.ex index d7eee0d1..d8f56bf5 100644 --- a/lib/supavisor/manager.ex +++ b/lib/supavisor/manager.ex @@ -3,9 +3,9 @@ defmodule Supavisor.Manager do use GenServer, restart: :transient require Logger + alias Supavisor.Helpers, as: H alias Supavisor.Protocol.Server alias Supavisor.Tenants - alias Supavisor.Helpers, as: H @check_timeout 120_000 @@ -135,7 +135,7 @@ defmodule Supavisor.Manager do ## Internal functions - defp check_subscribers() do + defp check_subscribers do Process.send_after( self(), :check_subscribers, @@ -143,7 +143,7 @@ defmodule Supavisor.Manager do ) end - defp now() do + defp now do System.system_time(:second) end diff --git a/lib/supavisor/monitoring/osmon.ex b/lib/supavisor/monitoring/osmon.ex index 75226a34..5a7cb4a4 100644 --- a/lib/supavisor/monitoring/osmon.ex +++ b/lib/supavisor/monitoring/osmon.ex @@ -61,7 +61,7 @@ defmodule Supavisor.PromEx.Plugins.OsMon do ) end - def execute_metrics() do + def execute_metrics do execute_metrics(@event_ram_usage, %{ram: ram_usage()}) execute_metrics(@event_cpu_util, %{cpu: cpu_util()}) execute_metrics(@event_cpu_la, cpu_la()) @@ -72,13 +72,13 @@ defmodule Supavisor.PromEx.Plugins.OsMon do end @spec ram_usage() :: float() - def ram_usage() do + def ram_usage do mem = :memsup.get_system_memory_data() 100 - mem[:free_memory] / mem[:total_memory] * 100 end @spec cpu_la() :: %{avg1: float(), avg5: float(), avg15: float()} - def cpu_la() do + def cpu_la do %{ avg1: :cpu_sup.avg1() / 256, avg5: :cpu_sup.avg5() / 256, @@ -87,7 +87,7 @@ defmodule Supavisor.PromEx.Plugins.OsMon do end @spec cpu_util() :: float() | {:error, term()} - def cpu_util() do + def cpu_util do :cpu_sup.util() end end diff --git a/lib/supavisor/monitoring/prom_ex.ex b/lib/supavisor/monitoring/prom_ex.ex index 01784463..c4524ce0 100644 --- a/lib/supavisor/monitoring/prom_ex.ex +++ b/lib/supavisor/monitoring/prom_ex.ex @@ -42,7 +42,7 @@ defmodule Supavisor.Monitoring.PromEx do end @spec set_metrics_tags() :: map() - def set_metrics_tags() do + def set_metrics_tags do [_, host] = node() |> Atom.to_string() |> String.split("@") metrics_tags = %{ @@ -61,7 +61,7 @@ defmodule Supavisor.Monitoring.PromEx do end @spec short_node_id() :: String.t() | nil - def short_node_id() do + def short_node_id do with {:ok, fly_alloc_id} when is_binary(fly_alloc_id) <- Application.fetch_env(:supavisor, :fly_alloc_id), [short_alloc_id, _] <- String.split(fly_alloc_id, "-", parts: 2) do @@ -72,7 +72,7 @@ defmodule Supavisor.Monitoring.PromEx do end @spec get_metrics() :: String.t() - def get_metrics() do + def get_metrics do metrics_tags = case Application.fetch_env(:supavisor, :metrics_tags) do :error -> set_metrics_tags() @@ -93,7 +93,7 @@ defmodule Supavisor.Monitoring.PromEx do end @spec do_cache_tenants_metrics() :: list - def do_cache_tenants_metrics() do + def do_cache_tenants_metrics do metrics = get_metrics() |> String.split("\n") pools = diff --git a/lib/supavisor/monitoring/telem.ex b/lib/supavisor/monitoring/telem.ex index 48c11328..71f36dea 100644 --- a/lib/supavisor/monitoring/telem.ex +++ b/lib/supavisor/monitoring/telem.ex @@ -14,12 +14,12 @@ defmodule Supavisor.Monitoring.Telem do end defmacro network_usage_disable(do: block) do - if not @metrics_disabled do - block - else + if @metrics_disabled do quote do {:ok, %{recv_oct: 0, send_oct: 0}} end + else + block end end diff --git a/lib/supavisor/monitoring/tenant.ex b/lib/supavisor/monitoring/tenant.ex index a4106cdc..ff976ab8 100644 --- a/lib/supavisor/monitoring/tenant.ex +++ b/lib/supavisor/monitoring/tenant.ex @@ -32,7 +32,7 @@ defmodule Supavisor.PromEx.Plugins.Tenant do buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000] end - defp client_metrics() do + defp client_metrics do Event.build( :supavisor_tenant_client_event_metrics, [ @@ -138,7 +138,7 @@ defmodule Supavisor.PromEx.Plugins.Tenant do ) end - defp db_metrics() do + defp db_metrics do Event.build( :supavisor_tenant_db_event_metrics, [ @@ -201,7 +201,7 @@ defmodule Supavisor.PromEx.Plugins.Tenant do ) end - def execute_tenant_metrics() do + def execute_tenant_metrics do Registry.select(Supavisor.Registry.TenantClients, [{{:"$1", :_, :_}, [], [:"$1"]}]) |> Enum.frequencies() |> Enum.each(&emit_telemetry_for_tenant/1) @@ -232,7 +232,7 @@ defmodule Supavisor.PromEx.Plugins.Tenant do ) end - def execute_conn_tenants_metrics() do + def execute_conn_tenants_metrics do num = Registry.select(Supavisor.Registry.TenantSups, [{{:"$1", :_, :_}, [], [:"$1"]}]) |> Enum.uniq() diff --git a/lib/supavisor/native_handler.ex b/lib/supavisor/native_handler.ex index a3de4d0a..db5f68b7 100644 --- a/lib/supavisor/native_handler.ex +++ b/lib/supavisor/native_handler.ex @@ -1,12 +1,15 @@ defmodule Supavisor.NativeHandler do @moduledoc false + use GenServer + @behaviour :ranch_protocol require Logger + alias Supavisor, as: S - alias Supavisor.Helpers, as: H alias Supavisor.HandlerHelpers, as: HH + alias Supavisor.Helpers, as: H alias Supavisor.{Protocol.Server, Tenants} @impl true @@ -149,7 +152,7 @@ defmodule Supavisor.NativeHandler do Registry.register(Supavisor.Registry.TenantClients, id, []) payload = - if !!hello.payload["user"] do + if hello.payload["user"] do %{hello.payload | "user" => user} else hello.payload @@ -161,7 +164,12 @@ defmodule Supavisor.NativeHandler do {:ok, addr} = HH.addr_from_sock(sock) - unless HH.filter_cidrs(tenant.allow_list, addr) == [] do + if HH.filter_cidrs(tenant.allow_list, addr) == [] do + message = "Address not in tenant allow_list: " <> inspect(addr) + Logger.error(message) + :ok = HH.send_error(sock, "XX000", message) + {:stop, :normal, state} + else case connect_local(host, port, payload, ip_ver, state.ssl) do {:ok, db_sock} -> auth = %{host: host, port: port, ip_ver: ip_ver} @@ -171,11 +179,6 @@ defmodule Supavisor.NativeHandler do Logger.error("Error connecting to tenant db: #{inspect(reason)}") {:stop, :normal, state} end - else - message = "Address not in tenant allow_list: " <> inspect(addr) - Logger.error(message) - :ok = HH.send_error(sock, "XX000", message) - {:stop, :normal, state} end _ -> diff --git a/lib/supavisor/protocol/client.ex b/lib/supavisor/protocol/client.ex index 307e114e..c13bd882 100644 --- a/lib/supavisor/protocol/client.ex +++ b/lib/supavisor/protocol/client.ex @@ -163,7 +163,7 @@ defmodule Supavisor.Protocol.Client do :undef end - def parse_msg_sel_1() do + def parse_msg_sel_1 do <<80, 0, 0, 0, 16, 0, 115, 101, 108, 101, 99, 116, 32, 49, 0, 0, 0, 66, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 6, 80, 0, 69, 0, 0, 0, 9, 0, 0, 0, 0, 200, 83, 0, 0, 0, 4>> end diff --git a/lib/supavisor/protocol/server.ex b/lib/supavisor/protocol/server.ex index c2986e24..d1e186cd 100644 --- a/lib/supavisor/protocol/server.ex +++ b/lib/supavisor/protocol/server.ex @@ -275,7 +275,7 @@ defmodule Supavisor.Protocol.Server do end @spec scram_request() :: iodata - def scram_request() do + def scram_request do @scram_request end @@ -323,11 +323,11 @@ defmodule Supavisor.Protocol.Server do decode_parameter_description(rest, [oid | acc]) end - def flush() do + def flush do <> end - def sync() do + def sync do <> end @@ -337,7 +337,7 @@ defmodule Supavisor.Protocol.Server do [<>, payload] end - def test_extended_query() do + def test_extended_query do [ encode("select * from todos where id = 40;"), [<<68, 0, 0, 0, 6, 83>>, [], <<0>>], @@ -345,13 +345,13 @@ defmodule Supavisor.Protocol.Server do ] end - def select_1_response() do + def select_1_response do <<84, 0, 0, 0, 33, 0, 1, 63, 99, 111, 108, 117, 109, 110, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4, 255, 255, 255, 255, 0, 0, 68, 0, 0, 0, 11, 0, 1, 0, 0, 0, 1, 49, 67, 0, 0, 0, 13, 83, 69, 76, 69, 67, 84, 32, 49, 0, 90, 0, 0, 0, 5, 73>> end - def authentication_ok() do + def authentication_ok do @authentication_ok end @@ -370,7 +370,7 @@ defmodule Supavisor.Protocol.Server do end @spec backend_key_data() :: {iodata(), binary} - def backend_key_data() do + def backend_key_data do pid = System.unique_integer([:positive, :monotonic]) key = :crypto.strong_rand_bytes(4) payload = <> @@ -379,13 +379,13 @@ defmodule Supavisor.Protocol.Server do end @spec ready_for_query() :: binary() - def ready_for_query() do + def ready_for_query do @ready_for_query end # SSLRequest message @spec ssl_request() :: binary() - def ssl_request() do + def ssl_request do @ssl_request end @@ -463,5 +463,5 @@ defmodule Supavisor.Protocol.Server do end @spec application_name() :: binary - def application_name(), do: @application_name + def application_name, do: @application_name end diff --git a/lib/supavisor/tenants.ex b/lib/supavisor/tenants.ex index 26095c67..d8ca5545 100644 --- a/lib/supavisor/tenants.ex +++ b/lib/supavisor/tenants.ex @@ -6,10 +6,10 @@ defmodule Supavisor.Tenants do import Ecto.Query, warn: false alias Supavisor.Repo - alias Supavisor.Tenants.Tenant - alias Supavisor.Tenants.User alias Supavisor.Tenants.Cluster alias Supavisor.Tenants.ClusterTenants + alias Supavisor.Tenants.Tenant + alias Supavisor.Tenants.User @doc """ Returns the list of tenants. diff --git a/lib/supavisor/tenants/cluster_tenants.ex b/lib/supavisor/tenants/cluster_tenants.ex index 0b247aa1..0abfe9ca 100644 --- a/lib/supavisor/tenants/cluster_tenants.ex +++ b/lib/supavisor/tenants/cluster_tenants.ex @@ -2,9 +2,11 @@ defmodule Supavisor.Tenants.ClusterTenants do @moduledoc false use Ecto.Schema + import Ecto.Changeset - alias Supavisor.Tenants.Tenant + alias Supavisor.Tenants.Cluster + alias Supavisor.Tenants.Tenant @type t :: %__MODULE__{} diff --git a/lib/supavisor/tenants_metrics.ex b/lib/supavisor/tenants_metrics.ex index 25f71083..973dc72f 100644 --- a/lib/supavisor/tenants_metrics.ex +++ b/lib/supavisor/tenants_metrics.ex @@ -41,7 +41,7 @@ defmodule Supavisor.TenantsMetrics do ## Internal functions - defp check_metrics() do + defp check_metrics do Process.send_after( self(), :check_metrics, diff --git a/lib/supavisor_web/api_spec.ex b/lib/supavisor_web/api_spec.ex index a86a089f..948fa947 100644 --- a/lib/supavisor_web/api_spec.ex +++ b/lib/supavisor_web/api_spec.ex @@ -4,8 +4,8 @@ defmodule SupavisorWeb.ApiSpec do alias OpenApiSpex.Info alias OpenApiSpex.OpenApi alias OpenApiSpex.Paths - alias OpenApiSpex.Server alias OpenApiSpex.SecurityScheme + alias OpenApiSpex.Server alias SupavisorWeb.Endpoint alias SupavisorWeb.Router diff --git a/lib/supavisor_web/controllers/metrics_controller.ex b/lib/supavisor_web/controllers/metrics_controller.ex index def7b538..cd34d67a 100644 --- a/lib/supavisor_web/controllers/metrics_controller.ex +++ b/lib/supavisor_web/controllers/metrics_controller.ex @@ -27,7 +27,7 @@ defmodule SupavisorWeb.MetricsController do end @spec fetch_cluster_metrics() :: String.t() - def fetch_cluster_metrics() do + def fetch_cluster_metrics do Node.list() |> Task.async_stream(&fetch_node_metrics/1, timeout: :infinity) |> Enum.reduce(PromEx.get_metrics(), &merge_node_metrics/2) diff --git a/lib/supavisor_web/controllers/tenant_controller.ex b/lib/supavisor_web/controllers/tenant_controller.ex index bf23ef83..4cdd14e6 100644 --- a/lib/supavisor_web/controllers/tenant_controller.ex +++ b/lib/supavisor_web/controllers/tenant_controller.ex @@ -4,16 +4,21 @@ defmodule SupavisorWeb.TenantController do require Logger - alias Supavisor.{Tenants, Repo, Helpers} + alias Supavisor.{ + Helpers, + Repo, + Tenants + } + alias Tenants.Tenant, as: TenantModel alias SupavisorWeb.OpenApiSchemas.{ + Created, + Empty, + NotFound, Tenant, - TenantList, TenantCreate, - NotFound, - Created, - Empty + TenantList } action_fallback(SupavisorWeb.FallbackController) diff --git a/lib/supavisor_web/open_api_schemas.ex b/lib/supavisor_web/open_api_schemas.ex index 37a44b1e..23798f5c 100644 --- a/lib/supavisor_web/open_api_schemas.ex +++ b/lib/supavisor_web/open_api_schemas.ex @@ -51,7 +51,7 @@ defmodule SupavisorWeb.OpenApiSchemas do } }) - def response(), do: {"User Response", "application/json", __MODULE__} + def response, do: {"User Response", "application/json", __MODULE__} end defmodule Tenant do @@ -114,7 +114,7 @@ defmodule SupavisorWeb.OpenApiSchemas do } }) - def response(), do: {"Tenant Response", "application/json", __MODULE__} + def response, do: {"Tenant Response", "application/json", __MODULE__} end defmodule TenantList do @@ -122,7 +122,7 @@ defmodule SupavisorWeb.OpenApiSchemas do require OpenApiSpex OpenApiSpex.schema(%{type: :array, items: Tenant}) - def response(), do: {"Tenant List Response", "application/json", __MODULE__} + def response, do: {"Tenant List Response", "application/json", __MODULE__} end defmodule TenantCreate do @@ -190,7 +190,7 @@ defmodule SupavisorWeb.OpenApiSchemas do required: [:tenant] }) - def params(), do: {"Tenant Create Params", "application/json", __MODULE__} + def params, do: {"Tenant Create Params", "application/json", __MODULE__} end defmodule Created do @@ -203,7 +203,7 @@ defmodule SupavisorWeb.OpenApiSchemas do require OpenApiSpex OpenApiSpex.schema(%{}) - def response(), do: {"", "application/json", __MODULE__} + def response, do: {"", "application/json", __MODULE__} end defmodule NotFound do @@ -211,6 +211,6 @@ defmodule SupavisorWeb.OpenApiSchemas do require OpenApiSpex OpenApiSpex.schema(%{}) - def response(), do: {"Not found", "application/json", __MODULE__} + def response, do: {"Not found", "application/json", __MODULE__} end end diff --git a/lib/supavisor_web/views/cluster_view.ex b/lib/supavisor_web/views/cluster_view.ex index 8478edd9..6073c635 100644 --- a/lib/supavisor_web/views/cluster_view.ex +++ b/lib/supavisor_web/views/cluster_view.ex @@ -1,7 +1,8 @@ defmodule SupavisorWeb.ClusterView do use SupavisorWeb, :view - alias SupavisorWeb.ClusterView + alias SupavisorWeb.ClusterTenantsView + alias SupavisorWeb.ClusterView def render("index.json", %{clusters: clusters}) do %{data: render_many(clusters, ClusterView, "cluster.json")} diff --git a/lib/supavisor_web/ws_proxy.ex b/lib/supavisor_web/ws_proxy.ex index ca5ba995..46a57377 100644 --- a/lib/supavisor_web/ws_proxy.ex +++ b/lib/supavisor_web/ws_proxy.ex @@ -59,7 +59,7 @@ defmodule SupavisorWeb.WsProxy do def filter_pass_pkt(bin), do: bin @spec connect_local() :: {:ok, port()} | {:error, term()} - defp connect_local() do + defp connect_local do proxy_port = Application.fetch_env!(:supavisor, :proxy_port_transaction) :gen_tcp.connect(~c"localhost", proxy_port, [:binary, packet: :raw, active: true]) end From 320dfb3510777f59533062e38e6b01b913e4adbd Mon Sep 17 00:00:00 2001 From: Stas Date: Thu, 22 Aug 2024 12:27:53 +0200 Subject: [PATCH 39/97] fix: handle prepared statements (#421) --- lib/supavisor.ex | 2 -- lib/supavisor/application.ex | 1 - lib/supavisor/client_handler.ex | 8 +++++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 8f6f1be1..e9b2c311 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -3,8 +3,6 @@ defmodule Supavisor do require Logger - import Cachex.Spec - alias Supavisor.{ Helpers, Manager, diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index ca0fbdd8..3b9802df 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -7,7 +7,6 @@ defmodule Supavisor.Application do require Logger - alias Supavisor.Handlers.Proxy.Handler, as: ProxyHandler alias Supavisor.Monitoring.PromEx @metrics_disabled Application.compile_env(:supavisor, :metrics_disabled, false) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 3324dd53..01c3e2b5 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -503,6 +503,7 @@ defmodule Supavisor.ClientHandler do when is_binary(bin) and is_pid(pid) do Logger.debug("ClientHandler: Receive query #{inspect(bin)}") db_pid = db_checkout(:both, :on_query, data) + handle_prepared_statements(db_pid, bin, data) {:next_state, :busy, %{data | db_pid: db_pid, query_start: System.monotonic_time()}, {:next_event, :internal, {proto, nil, bin}}} @@ -776,7 +777,8 @@ defmodule Supavisor.ClientHandler do else: {:error, "Wrong password"} end - @spec db_checkout(:write | :read | :both, :on_connect | :on_query, map) :: {pid, pid} | nil + @spec db_checkout(:write | :read | :both, :on_connect | :on_query, map) :: + {pid, pid, Supavisor.sock()} | nil defp db_checkout(_, _, %{mode: mode, db_pid: {pool, db_pid, db_sock}}) when is_pid(db_pid) and mode in [:session, :proxy] do {pool, db_pid, db_sock} @@ -991,8 +993,8 @@ defmodule Supavisor.ClientHandler do end end - @spec handle_prepared_statements({pid, pid}, binary, map) :: :ok | nil - defp handle_prepared_statements({_, pid}, bin, %{mode: :transaction} = data) do + @spec handle_prepared_statements({pid, pid, Supavisor.sock()}, binary, map) :: :ok | nil + defp handle_prepared_statements({_, pid, _}, bin, %{mode: :transaction} = data) do with {:ok, payload} <- Client.get_payload(bin), {:ok, statements} <- Supavisor.PgParser.statements(payload), true <- statements in [["PrepareStmt"], ["DeallocateStmt"]] do From 3c7d0752a58ee8add48afc9a8e9c93aee8315f09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Thu, 22 Aug 2024 12:35:22 +0200 Subject: [PATCH 40/97] fix: cleanup warnings (#420) --- flake.lock | 28 ++++++++++++++-------------- lib/supavisor/client_handler.ex | 5 ++++- lib/supavisor/manager.ex | 4 ++-- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/flake.lock b/flake.lock index 9722da48..d676e8df 100644 --- a/flake.lock +++ b/flake.lock @@ -41,11 +41,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1720699919, - "narHash": "sha256-5DNu5bWJbh3ZX6fWI4gJDsHcZlYCPSM7pWNMfoza+hA=", + "lastModified": 1723487333, + "narHash": "sha256-jqi/hVQL6S9lj/HkWaPPZQW/BfP0D0Veb45cpSvfRVE=", "owner": "cachix", "repo": "devenv", - "rev": "55106de9d798923df979e67811f8e1eda960c219", + "rev": "b285601679c7686f623791ad93a8e0debc322633", "type": "github" }, "original": { @@ -122,11 +122,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1719994518, - "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "lastModified": 1722555600, + "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", "type": "github" }, "original": { @@ -288,14 +288,14 @@ }, "nixpkgs-lib": { "locked": { - "lastModified": 1719876945, - "narHash": "sha256-Fm2rDDs86sHy0/1jxTOKB1118Q0O3Uc7EC0iXvXKpbI=", + "lastModified": 1722555339, + "narHash": "sha256-uFf2QeW7eAHlYXuDktm9c25OxOyCoUOQmh5SZ9amE5Q=", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/5daf0514482af3f97abaefc78a6606365c9108e2.tar.gz" + "url": "https://github.com/NixOS/nixpkgs/archive/a5d394176e64ab29c852d03346c1fc9b0b7d33eb.tar.gz" }, "original": { "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/5daf0514482af3f97abaefc78a6606365c9108e2.tar.gz" + "url": "https://github.com/NixOS/nixpkgs/archive/a5d394176e64ab29c852d03346c1fc9b0b7d33eb.tar.gz" } }, "nixpkgs-regression": { @@ -348,10 +348,10 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1720594544, - "narHash": "sha256-w6dlBUQYvS65f0Z33TvkcAj7ITr4NFqhF5ywss5T5bU=", - "path": "/nix/store/3x6hmlqc6682q9z6n11ynvmq19hfrgn0-source", - "rev": "aa9461550594533c29866d42f861b6ff079a7fb6", + "lastModified": 1723603349, + "narHash": "sha256-VMg6N7MryOuvSJ8Sj6YydarnUCkL7cvMdrMcnsJnJCE=", + "path": "/nix/store/qp204s0cpzbhj9yd5vpy7cpa9wxca0f9-source", + "rev": "daf7bb95821b789db24fc1ac21f613db0c1bf2cb", "type": "path" }, "original": { diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 01c3e2b5..21613f81 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -618,7 +618,10 @@ defmodule Supavisor.ClientHandler do {:next_state, :idle, %{data | db_pid: db_pid, stats: stats}, handle_actions(data)} :read_sql_error -> - Logger.error("ClientHandler: read only sql transaction, reruning the query to write pool") + Logger.error( + "ClientHandler: read only sql transaction, rerunning the query to write pool" + ) + # release the read pool _ = handle_db_pid(data.mode, data.pool, data.db_pid) diff --git a/lib/supavisor/manager.ex b/lib/supavisor/manager.ex index d8f56bf5..55c851e4 100644 --- a/lib/supavisor/manager.ex +++ b/lib/supavisor/manager.ex @@ -3,9 +3,9 @@ defmodule Supavisor.Manager do use GenServer, restart: :transient require Logger - alias Supavisor.Helpers, as: H alias Supavisor.Protocol.Server alias Supavisor.Tenants + alias Supavisor.Helpers @check_timeout 120_000 @@ -34,7 +34,7 @@ defmodule Supavisor.Manager do @impl true def init(args) do - H.set_log_level(args.log_level) + Helpers.set_log_level(args.log_level) tid = :ets.new(__MODULE__, [:protected]) [args | _] = Enum.filter(args.replicas, fn e -> e.replica_type == :write end) From 0d254fce851f06949f0b47c7a5073eafdd8cf2ec Mon Sep 17 00:00:00 2001 From: Stas Date: Thu, 22 Aug 2024 20:36:07 +0200 Subject: [PATCH 41/97] feat: ability to determine the pool node by aws zone (#422) --- Makefile | 2 ++ config/runtime.exs | 3 +- config/test.exs | 3 +- lib/supavisor.ex | 29 +++++++++++++++---- lib/supavisor/application.ex | 3 +- lib/supavisor/client_handler.ex | 11 +++++-- lib/supavisor/tenants/tenant.ex | 4 ++- .../20240822132419_add_aws_zone.exs | 9 ++++++ test/support/cluster.ex | 3 ++ 9 files changed, 54 insertions(+), 13 deletions(-) create mode 100644 priv/repo/migrations/20240822132419_add_aws_zone.exs diff --git a/Makefile b/Makefile index 11f3918c..1783a0c7 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ dev: CLUSTER_POSTGRES="true" \ DB_POOL_SIZE="5" \ METRICS_DISABLED="false" \ + AWS_ZONE="1b" \ ERL_AFLAGS="-kernel shell_history enabled +zdbbl 2097151" \ iex --name node1@127.0.0.1 --cookie cookie -S mix run --no-halt @@ -29,6 +30,7 @@ dev.node2: PROXY_PORT_TRANSACTION="6553" \ PROXY_PORT="5402" \ NODE_IP=localhost \ + AWS_ZONE="1c" \ ERL_AFLAGS="-kernel shell_history enabled" \ iex --name node2@127.0.0.1 --cookie cookie -S mix phx.server diff --git a/config/runtime.exs b/config/runtime.exs index ac72e94b..a52e96a4 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -159,7 +159,8 @@ if config_env() != :test do reconnect_on_db_close: System.get_env("RECONNECT_ON_DB_CLOSE") == "true", api_blocklist: System.get_env("API_TOKEN_BLOCKLIST", "") |> String.split(","), metrics_blocklist: System.get_env("METRICS_TOKEN_BLOCKLIST", "") |> String.split(","), - node_host: System.get_env("NODE_IP", "127.0.0.1") + node_host: System.get_env("NODE_IP", "127.0.0.1"), + aws_zone: System.get_env("AWS_ZONE") config :supavisor, Supavisor.Repo, url: System.get_env("DATABASE_URL", "ecto://postgres:postgres@localhost:6432/postgres"), diff --git a/config/test.exs b/config/test.exs index 0ba015ff..c4cb8e02 100644 --- a/config/test.exs +++ b/config/test.exs @@ -16,7 +16,8 @@ config :supavisor, "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJvbGUiOiJibG9ja2VkIiwiaWF0IjoxNjQ1MTkyODI0LCJleHAiOjE5NjA3Njg4MjR9.y-V3D1N2e8UTXc5PJzmV9cqMteq0ph2wl0yt42akQgA" ], metrics_blocklist: [], - node_host: System.get_env("NODE_IP", "127.0.0.1") + node_host: System.get_env("NODE_IP", "127.0.0.1"), + aws_zone: System.get_env("AWS_ZONE") config :supavisor, Supavisor.Repo, username: "postgres", diff --git a/lib/supavisor.ex b/lib/supavisor.ex index e9b2c311..f7451c33 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -22,13 +22,14 @@ defmodule Supavisor do @spec start_dist(id, secrets, keyword()) :: {:ok, pid()} | {:error, any()} def start_dist(id, secrets, options \\ []) do - options = Keyword.validate!(options, log_level: nil, force_node: false) + options = Keyword.validate!(options, log_level: nil, force_node: false, aws_zone: nil) log_level = Keyword.fetch!(options, :log_level) force_node = Keyword.fetch!(options, :force_node) + aws_zone = Keyword.fetch!(options, :aws_zone) case get_global_sup(id) do nil -> - node = if force_node, do: force_node, else: determine_node(id) + node = if force_node, do: force_node, else: determine_node(id, aws_zone) if node == node() do Logger.debug("Starting local pool for #{inspect(id)}") @@ -236,12 +237,28 @@ defmodule Supavisor do @spec mode(id) :: atom() def mode({_, _, mode, _}), do: mode - @spec determine_node(id) :: Node.t() - def determine_node(id) do + @spec determine_node(id, String.t() | nil) :: Node.t() + def determine_node(id, aws_zone) do tenant_id = tenant(id) - nodes = [node() | Node.list()] |> Enum.sort() + + # If the AWS zone group is empty, we will use all nodes. + # If the AWS zone group exists with the same zone, we will use nodes from this group. + # :syn.members(:aws_zone, "1c") + # [{#PID<0.381.0>, [node: :"node1@127.0.0.1"]}] + nodes = + with zone when is_binary(zone) <- aws_zone, + zone_nodes when zone_nodes != [] <- :syn.members(:aws_zone, zone) do + zone_nodes + |> Enum.map(fn {_, [node: node]} -> node end) + else + _ -> [node() | Node.list()] + end + index = :erlang.phash2(tenant_id, length(nodes)) - Enum.at(nodes, index) + + nodes + |> Enum.sort() + |> Enum.at(index) end @spec start_local_pool(id, secrets, atom()) :: {:ok, pid} | {:error, any} diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index 3b9802df..374a0ab7 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -60,7 +60,8 @@ defmodule Supavisor.Application do end :syn.set_event_handler(Supavisor.SynHandler) - :syn.add_node_to_scopes([:tenants]) + :syn.add_node_to_scopes([:tenants, :aws_zone]) + :syn.join(:aws_zone, Application.get_env(:supavisor, :aws_zone), self(), node: node()) topologies = Application.get_env(:libcluster, :topologies) || [] diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 21613f81..db2bc382 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -84,7 +84,8 @@ defmodule Supavisor.ClientHandler do connection_start: System.monotonic_time(), log_level: nil, db_sock: nil, - auth: %{} + auth: %{}, + tenant_aws_zone: nil } :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) @@ -373,7 +374,10 @@ defmodule Supavisor.ClientHandler do Logger.debug("ClientHandler: Subscribe to tenant #{inspect(data.id)}") with {:ok, sup} <- - Supavisor.start_dist(data.id, data.auth_secrets, log_level: data.log_level), + Supavisor.start_dist(data.id, data.auth_secrets, + log_level: data.log_level, + aws_zone: data.tenant_aws_zone + ), true <- if(node(sup) != node() and data.mode == :transaction, do: :proxy, else: true), {:ok, opts} <- Supavisor.subscribe(sup, data.id) do Process.monitor(opts.workers.manager) @@ -857,7 +861,8 @@ defmodule Supavisor.ClientHandler do heartbeat_interval: info.tenant.client_heartbeat_interval * 1000, db_name: db_name, mode: mode, - auth: auth + auth: auth, + tenant_aws_zone: info.tenant.aws_zone } end diff --git a/lib/supavisor/tenants/tenant.ex b/lib/supavisor/tenants/tenant.ex index 7864f2bd..0126de1a 100644 --- a/lib/supavisor/tenants/tenant.ex +++ b/lib/supavisor/tenants/tenant.ex @@ -31,6 +31,7 @@ defmodule Supavisor.Tenants.Tenant do field(:client_idle_timeout, :integer, default: 0) field(:client_heartbeat_interval, :integer, default: 60) field(:allow_list, {:array, :string}, default: ["0.0.0.0/0", "::/0"]) + field(:aws_zone, :string) has_many(:users, User, foreign_key: :tenant_external_id, @@ -63,7 +64,8 @@ defmodule Supavisor.Tenants.Tenant do :default_max_clients, :client_idle_timeout, :client_heartbeat_interval, - :allow_list + :allow_list, + :aws_zone ]) |> check_constraint(:upstream_ssl, name: :upstream_constraints, prefix: "_supavisor") |> check_constraint(:upstream_verify, name: :upstream_constraints, prefix: "_supavisor") diff --git a/priv/repo/migrations/20240822132419_add_aws_zone.exs b/priv/repo/migrations/20240822132419_add_aws_zone.exs new file mode 100644 index 00000000..7286b4b8 --- /dev/null +++ b/priv/repo/migrations/20240822132419_add_aws_zone.exs @@ -0,0 +1,9 @@ +defmodule Supavisor.Repo.Migrations.AddAwsZone do + use Ecto.Migration + + def change do + alter table("tenants", prefix: "_supavisor") do + add(:aws_zone, :string) + end + end +end diff --git a/test/support/cluster.ex b/test/support/cluster.ex index 921bf1f2..efd08d3e 100644 --- a/test/support/cluster.ex +++ b/test/support/cluster.ex @@ -37,6 +37,9 @@ defmodule Supavisor.Support.Cluster do {:supavisor, :region} -> "usa" + {:supavisor, :aws_zone} -> + "1c" + _ -> val end From c54b9c524cae9e392cc16996df6efb3cd511fc1d Mon Sep 17 00:00:00 2001 From: Stas Date: Thu, 22 Aug 2024 21:19:15 +0200 Subject: [PATCH 42/97] chore: rename aws_zone to availability_zone (#423) --- Makefile | 4 ++-- config/runtime.exs | 2 +- config/test.exs | 2 +- lib/supavisor.ex | 16 +++++++++------- lib/supavisor/application.ex | 7 +++++-- lib/supavisor/client_handler.ex | 6 +++--- lib/supavisor/tenants/tenant.ex | 4 ++-- ... => 20240822132419_add_availability_zone.exs} | 2 +- test/support/cluster.ex | 4 ++-- 9 files changed, 26 insertions(+), 21 deletions(-) rename priv/repo/migrations/{20240822132419_add_aws_zone.exs => 20240822132419_add_availability_zone.exs} (80%) diff --git a/Makefile b/Makefile index 1783a0c7..21a4a7e4 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ dev: CLUSTER_POSTGRES="true" \ DB_POOL_SIZE="5" \ METRICS_DISABLED="false" \ - AWS_ZONE="1b" \ + AVAILABILITY_ZONE="ap-southeast-1b" \ ERL_AFLAGS="-kernel shell_history enabled +zdbbl 2097151" \ iex --name node1@127.0.0.1 --cookie cookie -S mix run --no-halt @@ -30,7 +30,7 @@ dev.node2: PROXY_PORT_TRANSACTION="6553" \ PROXY_PORT="5402" \ NODE_IP=localhost \ - AWS_ZONE="1c" \ + AVAILABILITY_ZONE="ap-southeast-1c" \ ERL_AFLAGS="-kernel shell_history enabled" \ iex --name node2@127.0.0.1 --cookie cookie -S mix phx.server diff --git a/config/runtime.exs b/config/runtime.exs index a52e96a4..531f1cf4 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -160,7 +160,7 @@ if config_env() != :test do api_blocklist: System.get_env("API_TOKEN_BLOCKLIST", "") |> String.split(","), metrics_blocklist: System.get_env("METRICS_TOKEN_BLOCKLIST", "") |> String.split(","), node_host: System.get_env("NODE_IP", "127.0.0.1"), - aws_zone: System.get_env("AWS_ZONE") + availability_zone: System.get_env("AVAILABILITY_ZONE") config :supavisor, Supavisor.Repo, url: System.get_env("DATABASE_URL", "ecto://postgres:postgres@localhost:6432/postgres"), diff --git a/config/test.exs b/config/test.exs index c4cb8e02..e1aaaa74 100644 --- a/config/test.exs +++ b/config/test.exs @@ -17,7 +17,7 @@ config :supavisor, ], metrics_blocklist: [], node_host: System.get_env("NODE_IP", "127.0.0.1"), - aws_zone: System.get_env("AWS_ZONE") + availability_zone: System.get_env("AVAILABILITY_ZONE") config :supavisor, Supavisor.Repo, username: "postgres", diff --git a/lib/supavisor.ex b/lib/supavisor.ex index f7451c33..7a3cc5b0 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -22,14 +22,16 @@ defmodule Supavisor do @spec start_dist(id, secrets, keyword()) :: {:ok, pid()} | {:error, any()} def start_dist(id, secrets, options \\ []) do - options = Keyword.validate!(options, log_level: nil, force_node: false, aws_zone: nil) + options = + Keyword.validate!(options, log_level: nil, force_node: false, availability_zone: nil) + log_level = Keyword.fetch!(options, :log_level) force_node = Keyword.fetch!(options, :force_node) - aws_zone = Keyword.fetch!(options, :aws_zone) + availability_zone = Keyword.fetch!(options, :availability_zone) case get_global_sup(id) do nil -> - node = if force_node, do: force_node, else: determine_node(id, aws_zone) + node = if force_node, do: force_node, else: determine_node(id, availability_zone) if node == node() do Logger.debug("Starting local pool for #{inspect(id)}") @@ -238,16 +240,16 @@ defmodule Supavisor do def mode({_, _, mode, _}), do: mode @spec determine_node(id, String.t() | nil) :: Node.t() - def determine_node(id, aws_zone) do + def determine_node(id, availability_zone) do tenant_id = tenant(id) # If the AWS zone group is empty, we will use all nodes. # If the AWS zone group exists with the same zone, we will use nodes from this group. - # :syn.members(:aws_zone, "1c") + # :syn.members(:availability_zone, "1c") # [{#PID<0.381.0>, [node: :"node1@127.0.0.1"]}] nodes = - with zone when is_binary(zone) <- aws_zone, - zone_nodes when zone_nodes != [] <- :syn.members(:aws_zone, zone) do + with zone when is_binary(zone) <- availability_zone, + zone_nodes when zone_nodes != [] <- :syn.members(:availability_zone, zone) do zone_nodes |> Enum.map(fn {_, [node: node]} -> node end) else diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index 374a0ab7..69ea4555 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -60,8 +60,11 @@ defmodule Supavisor.Application do end :syn.set_event_handler(Supavisor.SynHandler) - :syn.add_node_to_scopes([:tenants, :aws_zone]) - :syn.join(:aws_zone, Application.get_env(:supavisor, :aws_zone), self(), node: node()) + :syn.add_node_to_scopes([:tenants, :availability_zone]) + + :syn.join(:availability_zone, Application.get_env(:supavisor, :availability_zone), self(), + node: node() + ) topologies = Application.get_env(:libcluster, :topologies) || [] diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index db2bc382..6147040c 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -85,7 +85,7 @@ defmodule Supavisor.ClientHandler do log_level: nil, db_sock: nil, auth: %{}, - tenant_aws_zone: nil + tenant_availability_zone: nil } :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) @@ -376,7 +376,7 @@ defmodule Supavisor.ClientHandler do with {:ok, sup} <- Supavisor.start_dist(data.id, data.auth_secrets, log_level: data.log_level, - aws_zone: data.tenant_aws_zone + availability_zone: data.tenant_availability_zone ), true <- if(node(sup) != node() and data.mode == :transaction, do: :proxy, else: true), {:ok, opts} <- Supavisor.subscribe(sup, data.id) do @@ -862,7 +862,7 @@ defmodule Supavisor.ClientHandler do db_name: db_name, mode: mode, auth: auth, - tenant_aws_zone: info.tenant.aws_zone + tenant_availability_zone: info.tenant.availability_zone } end diff --git a/lib/supavisor/tenants/tenant.ex b/lib/supavisor/tenants/tenant.ex index 0126de1a..a95d79d0 100644 --- a/lib/supavisor/tenants/tenant.ex +++ b/lib/supavisor/tenants/tenant.ex @@ -31,7 +31,7 @@ defmodule Supavisor.Tenants.Tenant do field(:client_idle_timeout, :integer, default: 0) field(:client_heartbeat_interval, :integer, default: 60) field(:allow_list, {:array, :string}, default: ["0.0.0.0/0", "::/0"]) - field(:aws_zone, :string) + field(:availability_zone, :string) has_many(:users, User, foreign_key: :tenant_external_id, @@ -65,7 +65,7 @@ defmodule Supavisor.Tenants.Tenant do :client_idle_timeout, :client_heartbeat_interval, :allow_list, - :aws_zone + :availability_zone ]) |> check_constraint(:upstream_ssl, name: :upstream_constraints, prefix: "_supavisor") |> check_constraint(:upstream_verify, name: :upstream_constraints, prefix: "_supavisor") diff --git a/priv/repo/migrations/20240822132419_add_aws_zone.exs b/priv/repo/migrations/20240822132419_add_availability_zone.exs similarity index 80% rename from priv/repo/migrations/20240822132419_add_aws_zone.exs rename to priv/repo/migrations/20240822132419_add_availability_zone.exs index 7286b4b8..bda994a2 100644 --- a/priv/repo/migrations/20240822132419_add_aws_zone.exs +++ b/priv/repo/migrations/20240822132419_add_availability_zone.exs @@ -3,7 +3,7 @@ defmodule Supavisor.Repo.Migrations.AddAwsZone do def change do alter table("tenants", prefix: "_supavisor") do - add(:aws_zone, :string) + add(:availability_zone, :string) end end end diff --git a/test/support/cluster.ex b/test/support/cluster.ex index efd08d3e..f2add46a 100644 --- a/test/support/cluster.ex +++ b/test/support/cluster.ex @@ -37,8 +37,8 @@ defmodule Supavisor.Support.Cluster do {:supavisor, :region} -> "usa" - {:supavisor, :aws_zone} -> - "1c" + {:supavisor, :availability_zone} -> + "ap-southeast-1c" _ -> val From 38982c4f432c8e862c67719ba986955740658549 Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 23 Aug 2024 12:12:31 +0200 Subject: [PATCH 43/97] feat: show availability_zone in logs metadata if it is available (#424) --- lib/supavisor/application.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index 69ea4555..655c5aa9 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -19,7 +19,10 @@ defmodule Supavisor.Application do :logger.set_primary_config( :metadata, Enum.into( - [region: System.get_env("REGION"), instance_id: System.get_env("INSTANCE_ID")], + [ + region: System.get_env("AVAILABILITY_ZONE") || System.get_env("REGION"), + instance_id: System.get_env("INSTANCE_ID") + ], primary_config.metadata ) ) From 8aeee4e1bf30e0b197474423abc09913f6731dec Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 23 Aug 2024 15:49:30 +0200 Subject: [PATCH 44/97] fix: set active false before try_ssl_handshake() (#425) --- lib/supavisor/db_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 664a5686..29eec411 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -98,7 +98,7 @@ defmodule Supavisor.DbHandler do # keepalive: true, # nopush: true, nodelay: true, - active: true + active: false ] reconnect_callback = {:keep_state_and_data, {:state_timeout, @reconnect_timeout, :connect}} From 19d59802c9fc45e452762b4bfbe0e80aa64db0bd Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 23 Aug 2024 15:49:55 +0200 Subject: [PATCH 45/97] fix: canceling statement (#426) --- lib/supavisor/client_handler.ex | 2 +- lib/supavisor/db_handler.ex | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 6147040c..4a43cb38 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -990,7 +990,7 @@ defmodule Supavisor.ClientHandler do def try_get_sni(_), do: nil - defp db_pid_meta({_, {_, pid}} = _key) do + defp db_pid_meta({_, {_, pid, _}} = _key) do rkey = Supavisor.Registry.PoolPids fnode = node(pid) diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 29eec411..7edc93ce 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -505,8 +505,13 @@ defmodule Supavisor.DbHandler do defp handle_auth_pkts(%{tag: :ready_for_query, payload: db_state}, acc, _), do: {:ready_for_query, Map.put(acc, :db_state, db_state)} - defp handle_auth_pkts(%{tag: :backend_key_data, payload: payload}, acc, _), - do: Map.put(acc, :backend_key_data, payload) + defp handle_auth_pkts(%{tag: :backend_key_data, payload: payload}, acc, data) do + key = self() + conn = %{host: data.auth.host, port: data.auth.port, ip_ver: data.auth.ip_version} + Registry.register(Supavisor.Registry.PoolPids, key, Map.merge(payload, conn)) + Logger.debug("DbHandler: Backend #{inspect(key)} data: #{inspect(payload)}") + Map.put(acc, :backend_key_data, payload) + end defp handle_auth_pkts(%{payload: {:authentication_sasl_password, methods_b}}, _, data) do nonce = From b5f37c42b6c2a0a78ff4baf901e235ae2fd78870 Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 23 Aug 2024 16:19:27 +0200 Subject: [PATCH 46/97] fix: sni_hostname typo (#427) --- lib/supavisor/db_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 7edc93ce..cd30da5e 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -408,7 +408,7 @@ defmodule Supavisor.DbHandler do verify: :verify_peer, cacerts: [auth.upstream_tls_ca], # unclear behavior on pg14 - server_name_indication: auth.sni_host || auth.host, + server_name_indication: auth.sni_hostname || auth.host, customize_hostname_check: [{:match_fun, fn _, _ -> true end}] ] From ecafdcfb56da98a0d4010c1dfb3dc22c0c989351 Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 23 Aug 2024 16:52:35 +0200 Subject: [PATCH 47/97] fix: convert sni to charlsit (#428) --- lib/supavisor.ex | 2 ++ lib/supavisor/client_handler.ex | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 7a3cc5b0..9c24ec46 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -321,6 +321,7 @@ defmodule Supavisor do default_max_clients: def_max_clients, client_idle_timeout: client_idle_timeout, replica_type: replica_type, + sni_hostname: sni_hostname, users: [ %{ db_user: db_user, @@ -341,6 +342,7 @@ defmodule Supavisor do auth = %{ host: String.to_charlist(db_host), + sni_hostname: if(sni_hostname != nil, do: to_charlist(sni_hostname)), port: db_port, user: db_user, database: if(db_name != nil, do: db_name, else: db_database), diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 4a43cb38..824a69ad 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -839,7 +839,8 @@ defmodule Supavisor.ClientHandler do application_name: data[:app_name] || "Supavisor", database: info.tenant.db_database, host: to_charlist(info.tenant.db_host), - sni_host: info.tenant.sni_hostname, + sni_hostname: + if(info.tenant.sni_hostname != nil, do: to_charlist(info.tenant.sni_hostname)), ip_version: Helpers.ip_version(info.tenant.ip_version, info.tenant.db_host), port: info.tenant.db_port, user: user, From da8414471669d7e811fc13608d8e4566eb06264a Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 23 Aug 2024 18:08:32 +0200 Subject: [PATCH 48/97] fix: ignore ssl enforcement for proxy (#429) --- lib/supavisor/client_handler.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 824a69ad..121477a2 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -85,7 +85,8 @@ defmodule Supavisor.ClientHandler do log_level: nil, db_sock: nil, auth: %{}, - tenant_availability_zone: nil + tenant_availability_zone: nil, + local: opts[:local] || false } :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) @@ -242,7 +243,7 @@ defmodule Supavisor.ClientHandler do {:ok, addr} = HandlerHelpers.addr_from_sock(sock) cond do - info.tenant.enforce_ssl and !data.ssl -> + !data.local and info.tenant.enforce_ssl and !data.ssl -> Logger.error( "ClientHandler: Tenant is not allowed to connect without SSL, user #{user}" ) From 51c0aadd9f194673157a0fba1f6508d58b30806d Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 23 Aug 2024 20:41:25 +0200 Subject: [PATCH 49/97] chore: change log level for some pg messages (#430) --- lib/supavisor.ex | 2 +- lib/supavisor/client_handler.ex | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 9c24ec46..859e8758 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -265,7 +265,7 @@ defmodule Supavisor do @spec start_local_pool(id, secrets, atom()) :: {:ok, pid} | {:error, any} def start_local_pool({{type, tenant}, _user, _mode, _db_name} = id, secrets, log_level \\ nil) do - Logger.debug("Starting pool(s) for #{inspect(id)}") + Logger.info("Starting pool(s) for #{inspect(id)}") user = elem(secrets, 1).().alias diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 121477a2..77c5e901 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -477,28 +477,28 @@ defmodule Supavisor.ClientHandler do # handle Terminate message def handle_event(:info, {proto, _, <>}, :idle, _) when proto in @proto do - Logger.error("ClientHandler: Terminate received from client") + Logger.info("ClientHandler: Terminate received from client") {:stop, {:shutdown, :terminate_received}} end # handle Sync message def handle_event(:info, {proto, _, <>}, :idle, data) when proto in @proto do - Logger.error("ClientHandler: Receive sync") + Logger.info("ClientHandler: Receive sync") :ok = HandlerHelpers.sock_send(data.sock, Server.ready_for_query()) {:keep_state_and_data, handle_actions(data)} end def handle_event(:info, {proto, _, <> = msg}, _, data) when proto in @proto do - Logger.error("ClientHandler: Receive sync while not idle") + Logger.warning("ClientHandler: Receive sync while not idle") :ok = HandlerHelpers.sock_send(elem(data.db_pid, 2), msg) :keep_state_and_data end def handle_event(:info, {proto, _, <> = msg}, _, data) when proto in @proto do - Logger.error("ClientHandler: Receive flush while not idle") + Logger.warning("ClientHandler: Receive flush while not idle") :ok = HandlerHelpers.sock_send(elem(data.db_pid, 2), msg) :keep_state_and_data end From 895ffef338d08e5a8fb44ca7bf0bc943fb0911eb Mon Sep 17 00:00:00 2001 From: Stas Date: Tue, 27 Aug 2024 10:39:20 +0200 Subject: [PATCH 50/97] feat: limit the number of pools per tenant (#432) --- config/test.exs | 3 +- lib/supavisor.ex | 18 +++++- lib/supavisor/client_handler.ex | 7 +++ priv/repo/seeds_after_migration.exs | 2 +- test/integration/proxy_test.exs | 90 ++++++++++++++++++++++++----- 5 files changed, 102 insertions(+), 18 deletions(-) diff --git a/config/test.exs b/config/test.exs index e1aaaa74..41704b37 100644 --- a/config/test.exs +++ b/config/test.exs @@ -17,7 +17,8 @@ config :supavisor, ], metrics_blocklist: [], node_host: System.get_env("NODE_IP", "127.0.0.1"), - availability_zone: System.get_env("AVAILABILITY_ZONE") + availability_zone: System.get_env("AVAILABILITY_ZONE"), + max_pools: 3 config :supavisor, Supavisor.Repo, username: "postgres", diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 859e8758..1ec3886e 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -19,6 +19,7 @@ defmodule Supavisor do @type subscribe_opts :: %{workers: workers, ps: list, idle_timeout: integer} @registry Supavisor.Registry.Tenants + @max_pools Application.compile_env(:supavisor, :max_pools, 10) @spec start_dist(id, secrets, keyword()) :: {:ok, pid()} | {:error, any()} def start_dist(id, secrets, options \\ []) do @@ -35,10 +36,10 @@ defmodule Supavisor do if node == node() do Logger.debug("Starting local pool for #{inspect(id)}") - start_local_pool(id, secrets, log_level) + try_start_local_pool(id, secrets, log_level) else Logger.debug("Starting remote pool for #{inspect(id)}") - Helpers.rpc(node, __MODULE__, :start_local_pool, [id, secrets, log_level]) + Helpers.rpc(node, __MODULE__, :try_start_local_pool, [id, secrets, log_level]) end pid -> @@ -50,7 +51,7 @@ defmodule Supavisor do def start(id, secrets) do case get_global_sup(id) do nil -> - start_local_pool(id, secrets) + try_start_local_pool(id, secrets, nil) pid -> {:ok, pid} @@ -263,6 +264,13 @@ defmodule Supavisor do |> Enum.at(index) end + @spec try_start_local_pool(id, secrets, atom()) :: {:ok, pid} | {:error, any} + def try_start_local_pool(id, secrets, log_level) do + if count_pools(tenant(id)) < @max_pools, + do: start_local_pool(id, secrets, log_level), + else: {:error, :max_pools_reached} + end + @spec start_local_pool(id, secrets, atom()) :: {:ok, pid} | {:error, any} def start_local_pool({{type, tenant}, _user, _mode, _db_name} = id, secrets, log_level \\ nil) do Logger.info("Starting pool(s) for #{inspect(id)}") @@ -411,4 +419,8 @@ defmodule Supavisor do {:ok, %{listener: pid, host: host, port: :ranch.get_port(args.id)}} end end + + @spec count_pools(String.t()) :: non_neg_integer() + def count_pools(tenant), + do: Registry.count_match(Supavisor.Registry.TenantSups, tenant, :_) end diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 77c5e901..8afcb5da 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -400,6 +400,13 @@ defmodule Supavisor.ClientHandler do Telem.client_join(:fail, data.id) {:stop, {:shutdown, :max_clients_reached}} + {:error, :max_pools_reached} -> + msg = "Max pools count reached" + Logger.error("ClientHandler: #{msg}") + :ok = HandlerHelpers.send_error(data.sock, "XX000", msg) + Telem.client_join(:fail, data.id) + {:stop, {:shutdown, :max_pools_reached}} + :proxy -> {:ok, %{port: port, host: host}} = Supavisor.get_pool_ranch(data.id) diff --git a/priv/repo/seeds_after_migration.exs b/priv/repo/seeds_after_migration.exs index 51873ecd..e244c472 100644 --- a/priv/repo/seeds_after_migration.exs +++ b/priv/repo/seeds_after_migration.exs @@ -39,7 +39,7 @@ if !Tenants.get_tenant_by_external_id("is_manager") do |> Tenants.create_tenant() end -["proxy_tenant1", "syn_tenant", "prom_tenant"] +["proxy_tenant1", "syn_tenant", "prom_tenant", "max_pool_tenant"] |> Enum.each(fn tenant -> if !Tenants.get_tenant_by_external_id(tenant) do %{ diff --git a/test/integration/proxy_test.exs b/test/integration/proxy_test.exs index 1b90d1b1..646d0e62 100644 --- a/test/integration/proxy_test.exs +++ b/test/integration/proxy_test.exs @@ -155,19 +155,28 @@ defmodule Supavisor.Integration.ProxyTest do P.query!(origin, "select * from public.test where details = 'test_delete'", []) end - # test "too many clients in session mode" do - # db_conf = Application.get_env(:supavisor, Repo) - - # url = - # "postgresql://session.#{@tenant}:#{db_conf[:password]}@#{db_conf[:hostname]}:#{Application.get_env(:supavisor, :proxy_port)}/postgres" - - # spawn(fn -> System.cmd("psql", [url], stderr_to_stdout: true) end) + test "too many clients in session mode" do + Process.flag(:trap_exit, true) + db_conf = Application.get_env(:supavisor, Repo) + port = Application.get_env(:supavisor, :proxy_port_session) - # :timer.sleep(500) + connection_opts = + Keyword.merge(db_conf, + username: "max_clients.proxy_tenant1", + port: port + ) - # {result, _} = System.cmd("psql", [url], stderr_to_stdout: true) - # assert result =~ "FATAL: Too many clients already" - # end + assert {:error, + %Postgrex.Error{ + postgres: %{ + code: :internal_error, + message: "Max client connections reached", + unknown: "FATAL", + severity: "FATAL", + pg_code: "XX000" + } + }} = single_connection(connection_opts) + end test "http to proxy server returns 200 OK" do assert :httpc.request( @@ -206,7 +215,7 @@ defmodule Supavisor.Integration.ProxyTest do connection_opts = [ hostname: db_conf[:hostname], port: Application.get_env(:supavisor, :proxy_port_transaction), - username: "max_clients.#{@tenant}", + username: "max_clients.prom_tenant", database: "postgres", password: db_conf[:password] ] @@ -237,7 +246,7 @@ defmodule Supavisor.Integration.ProxyTest do assert {:ok, pid} = parse_uri(first_pass) |> single_connection() assert [%Postgrex.Result{rows: [["1"]]}] = P.SimpleConnection.call(pid, {:query, "select 1;"}) - + :gen_statem.stop(pid) P.query(origin, "alter user dev_postgres with password 'postgres_new';", []) Supavisor.stop({{:single, "is_manager"}, "dev_postgres", :transaction, "postgres"}) @@ -250,6 +259,7 @@ defmodule Supavisor.Integration.ProxyTest do {:ok, pid} = parse_uri(new_pass) |> single_connection() assert [%Postgrex.Result{rows: [["1"]]}] = P.SimpleConnection.call(pid, {:query, "select 1;"}) + :gen_statem.stop(pid) end test "invalid characters in user or db_name" do @@ -271,6 +281,60 @@ defmodule Supavisor.Integration.ProxyTest do }} = parse_uri(url) |> single_connection() end + test "max_pools limit" do + Process.flag(:trap_exit, true) + db_conf = Application.get_env(:supavisor, Repo) + port = Application.get_env(:supavisor, :proxy_port_transaction) + + tenant = "max_pool_tenant" + + {:ok, pid1} = + Keyword.merge(db_conf, + username: "postgres.#{tenant}", + port: port + ) + |> single_connection() + + assert Supavisor.count_pools(tenant) == 1 + + {:ok, pid2} = + Keyword.merge(db_conf, + username: "session.#{tenant}", + port: port + ) + |> single_connection() + + assert Supavisor.count_pools(tenant) == 2 + + {:ok, pid3} = + Keyword.merge(db_conf, + username: "transaction.#{tenant}", + port: port + ) + |> single_connection() + + assert Supavisor.count_pools(tenant) == 3 + + connection_opts = + Keyword.merge(db_conf, + username: "max_clients.#{tenant}", + port: port + ) + + assert {:error, + %Postgrex.Error{ + postgres: %{ + code: :internal_error, + message: "Max pools count reached", + unknown: "FATAL", + severity: "FATAL", + pg_code: "XX000" + } + }} = single_connection(connection_opts) + + for pid <- [pid1, pid2, pid3], do: :gen_statem.stop(pid) + end + defp single_connection(db_conf, c_port \\ nil) when is_list(db_conf) do port = c_port || db_conf[:port] From d08824b59836f972ba1a77020168cc1a95543efa Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 28 Aug 2024 12:12:21 +0200 Subject: [PATCH 51/97] feat: disable active socket during long queries (#434) --- config/config.exs | 3 ++- lib/supavisor/client_handler.ex | 18 +++++++++++++----- lib/supavisor/db_handler.ex | 16 ++++++++++++---- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/config/config.exs b/config/config.exs index 9fbf27db..28bacc51 100644 --- a/config/config.exs +++ b/config/config.exs @@ -11,7 +11,8 @@ config :supavisor, ecto_repos: [Supavisor.Repo], version: Mix.Project.config()[:version], env: Mix.env(), - metrics_disabled: System.get_env("METRICS_DISABLED") == "true" + metrics_disabled: System.get_env("METRICS_DISABLED") == "true", + switch_active_count: System.get_env("SWITCH_ACTIVE_COUNT", "100") |> String.to_integer() # Configures the endpoint config :supavisor, SupavisorWeb.Endpoint, diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 8afcb5da..b67dcd1a 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -12,6 +12,7 @@ defmodule Supavisor.ClientHandler do @behaviour :gen_statem @proto [:tcp, :ssl] @cancel_query_msg <<16::32, 1234::16, 5678::16>> + @switch_active_count Application.compile_env(:supavisor, :switch_active_count) alias Supavisor.{ DbHandler, @@ -86,7 +87,8 @@ defmodule Supavisor.ClientHandler do db_sock: nil, auth: %{}, tenant_availability_zone: nil, - local: opts[:local] || false + local: opts[:local] || false, + active_count: 0 } :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) @@ -555,13 +557,14 @@ defmodule Supavisor.ClientHandler do # forward query to db def handle_event(_, {proto, _, bin}, :busy, data) when proto in @proto do - # HandlerHelpers.setopts(data.sock, active: :once) - Logger.debug("ClientHandler: Forward query to db #{inspect(bin)} #{inspect(data.db_pid)}") + if data.active_count > @switch_active_count, + do: HandlerHelpers.active_once(data.sock) + case HandlerHelpers.sock_send(elem(data.db_pid, 2), bin) do :ok -> - :keep_state_and_data + {:keep_state, %{data | active_count: data.active_count + 1}} error -> Logger.error("ClientHandler: error while sending query: #{inspect(error)}") @@ -627,7 +630,12 @@ defmodule Supavisor.ClientHandler do {_, stats} = Telem.network_usage(:client, data.sock, data.id, data.stats) Telem.client_query_time(data.query_start, data.id) - {:next_state, :idle, %{data | db_pid: db_pid, stats: stats}, handle_actions(data)} + + if data.active_count > @switch_active_count, + do: HandlerHelpers.activate(data.sock) + + {:next_state, :idle, %{data | db_pid: db_pid, stats: stats, active_count: 0}, + handle_actions(data)} :read_sql_error -> Logger.error( diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index cd30da5e..b54191b2 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -21,6 +21,7 @@ defmodule Supavisor.DbHandler do @reconnect_timeout 2_500 @sock_closed [:tcp_closed, :ssl_closed] @proto [:tcp, :ssl] + @switch_active_count Application.compile_env(:supavisor, :switch_active_count) def start_link(config), do: :gen_statem.start_link(__MODULE__, config, hibernate_after: 5_000) @@ -72,7 +73,8 @@ defmodule Supavisor.DbHandler do pool: Supavisor.get_local_pool(args.id), caller: args[:caller] || nil, client_sock: args[:client_sock] || nil, - proxy: args[:proxy] || false + proxy: args[:proxy] || false, + active_count: 0 } Telem.handler_action(:db_handler, :started, args.id) @@ -279,22 +281,28 @@ defmodule Supavisor.DbHandler do when is_pid(caller) and proto in @proto do Logger.debug("DbHandler: Got write replica message #{inspect(bin)}") + if data.active_count > @switch_active_count, + do: HandlerHelpers.active_once(data.sock) + if String.ends_with?(bin, Server.ready_for_query()) do {_, stats} = Telem.network_usage(:db, data.sock, data.id, data.stats) data = if data.mode == :transaction do ClientHandler.db_status(data.caller, :ready_for_query, bin) - %{data | stats: stats, caller: nil, client_sock: nil} + %{data | stats: stats, caller: nil, client_sock: nil, active_count: 0} else HandlerHelpers.sock_send(data.client_sock, bin) - %{data | stats: stats} + %{data | stats: stats, active_count: 0} end + if data.active_count > @switch_active_count, + do: HandlerHelpers.activate(data.sock) + {:next_state, :idle, data, {:next_event, :internal, :check_anon_buffer}} else HandlerHelpers.sock_send(data.client_sock, bin) - :keep_state_and_data + {:keep_state, %{data | active_count: data.active_count + 1}} end end From af737a51d84c046791171f2a1673f6cd7fbff12e Mon Sep 17 00:00:00 2001 From: Stas Date: Thu, 29 Aug 2024 16:03:20 +0200 Subject: [PATCH 52/97] feat: add a background metrics cleaner (#435) --- lib/supavisor/application.ex | 2 +- lib/supavisor/metrics_cleaner.ex | 57 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 lib/supavisor/metrics_cleaner.ex diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index 655c5aa9..85c7a462 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -101,7 +101,7 @@ defmodule Supavisor.Application do children else PromEx.set_metrics_tags() - children ++ [PromEx, Supavisor.TenantsMetrics] + children ++ [PromEx, Supavisor.TenantsMetrics, Supavisor.MetricsCleaner] end # start Cachex only if the node uses names, this is necessary for test setup diff --git a/lib/supavisor/metrics_cleaner.ex b/lib/supavisor/metrics_cleaner.ex new file mode 100644 index 00000000..2cb82258 --- /dev/null +++ b/lib/supavisor/metrics_cleaner.ex @@ -0,0 +1,57 @@ +defmodule Supavisor.MetricsCleaner do + @moduledoc false + + use GenServer + require Logger + + @interval :timer.minutes(30) + + def start_link(args), + do: GenServer.start_link(__MODULE__, args, name: __MODULE__) + + def init(_args) do + Logger.info("Starting MetricsCleaner") + {:ok, %{check_ref: check()}} + end + + def handle_info(:check, state) do + Process.cancel_timer(state.check_ref) + + start = System.monotonic_time(:millisecond) + loop_and_cleanup_metrics_table() + exec_time = System.monotonic_time(:millisecond) - start + + if exec_time > :timer.seconds(5), + do: Logger.warning("Metrics check took: #{exec_time} ms") + + {:noreply, %{state | check_ref: check()}} + end + + def handle_info(msg, state) do + Logger.error("Unexpected message: #{inspect(msg)}") + {:noreply, state} + end + + def check, do: Process.send_after(self(), :check, @interval) + + def loop_and_cleanup_metrics_table do + metrics_table = Supavisor.Monitoring.PromEx.Metrics + tenant_registry_table = :syn_registry_by_name_tenants + + fn + {{_, %{type: type, mode: mode, user: user, tenant: tenant, db_name: db}} = key, _}, _ -> + case :ets.lookup(tenant_registry_table, {{type, tenant}, user, mode, db}) do + [] -> + Logger.warning("Found orphaned metric: #{inspect(key)}") + :ets.delete(metrics_table, key) + + _ -> + nil + end + + _, acc -> + acc + end + |> :ets.foldl(nil, metrics_table) + end +end From fd8fe79d28be087bef8a50a5950c70eb90aabac2 Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 30 Aug 2024 15:42:52 +0200 Subject: [PATCH 53/97] feat: change max_pools to 20 (#436) --- lib/supavisor.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 1ec3886e..d115e830 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -19,7 +19,7 @@ defmodule Supavisor do @type subscribe_opts :: %{workers: workers, ps: list, idle_timeout: integer} @registry Supavisor.Registry.Tenants - @max_pools Application.compile_env(:supavisor, :max_pools, 10) + @max_pools Application.compile_env(:supavisor, :max_pools, 20) @spec start_dist(id, secrets, keyword()) :: {:ok, pid()} | {:error, any()} def start_dist(id, secrets, options \\ []) do From 43aeb60d8fc05d38d1e0022c9b5fcf18341d1664 Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 30 Aug 2024 18:38:09 +0200 Subject: [PATCH 54/97] fix: session mode during proxy (#437) --- config/config.exs | 3 +- config/dev.exs | 14 ++++++- lib/supavisor/client_handler.ex | 66 +++++++++++++++++++++--------- lib/supavisor/db_handler.ex | 1 - lib/supavisor/tenant_supervisor.ex | 3 +- 5 files changed, 64 insertions(+), 23 deletions(-) diff --git a/config/config.exs b/config/config.exs index 28bacc51..d165af37 100644 --- a/config/config.exs +++ b/config/config.exs @@ -34,7 +34,8 @@ config :logger, :console, :mode, :type, :app_name, - :peer_ip + :peer_ip, + :local ] # Use Jason for JSON parsing in Phoenix diff --git a/config/dev.exs b/config/dev.exs index af12753d..eefe173f 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -63,7 +63,19 @@ config :logger, :console, format: "$time [$level] $message $metadata\n", level: :debug, # level: :notice, - metadata: [:error_code, :file, :line, :pid, :project, :user, :mode, :type, :app_name, :peer_ip] + metadata: [ + :error_code, + :file, + :line, + :pid, + :project, + :user, + :mode, + :type, + :app_name, + :peer_ip, + :local + ] # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive. diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index b67dcd1a..e5cbb605 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -88,7 +88,9 @@ defmodule Supavisor.ClientHandler do auth: %{}, tenant_availability_zone: nil, local: opts[:local] || false, - active_count: 0 + active_count: 0, + peer_ip: Helpers.peer_ip(sock), + app_name: nil } :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) @@ -175,12 +177,22 @@ defmodule Supavisor.ClientHandler do Logger.debug("ClientHandler: Client startup message: #{inspect(hello)}") {type, {user, tenant_or_alias, db_name}} = HandlerHelpers.parse_user_info(hello.payload) - not_allowed = ["\"", "\\"] + # Validate user and db_name according to PostgreSQL rules. + # The rules are: 1-63 characters, alphanumeric, underscore and $ + # TODO: spaces are allowed in db_name, but we don't support it yet + rule = ~r/^[a-z_][a-z0-9_$]*$/ - if String.contains?(user, not_allowed) or String.contains?(db_name, not_allowed) do + if user =~ rule and db_name =~ rule do + log_level = maybe_change_log(hello) + event = {:hello, {type, {user, tenant_or_alias, db_name}}} + app_name = app_name(hello.payload["application_name"]) + + {:keep_state, %{data | log_level: log_level, app_name: app_name}, + {:next_event, :internal, event}} + else reason = "Invalid format for user or db_name" Logger.error("ClientHandler: #{inspect(reason)}") - Telem.client_join(:fail, data.id) + Telem.client_join(:fail, tenant_or_alias) HandlerHelpers.send_error( data.sock, @@ -188,18 +200,7 @@ defmodule Supavisor.ClientHandler do "Authentication error, reason: #{inspect(reason)}" ) - {:stop, {:shutdown, :invalid_characters}} - else - log_level = - case hello.payload["options"]["log_level"] do - nil -> nil - level -> String.to_existing_atom(level) - end - - Helpers.set_log_level(log_level) - - {:keep_state, %{data | log_level: log_level}, - {:next_event, :internal, {:hello, {type, {user, tenant_or_alias, db_name}}}}} + {:stop, {:shutdown, :invalid_format}} end {:error, error} -> @@ -237,7 +238,10 @@ defmodule Supavisor.ClientHandler do user: user, mode: mode, type: type, - db_name: db_name + db_name: db_name, + app_name: data.app_name, + peer_ip: data.peer_ip, + local: data.local ) Registry.register(Supavisor.Registry.TenantClients, id, []) @@ -381,7 +385,11 @@ defmodule Supavisor.ClientHandler do log_level: data.log_level, availability_zone: data.tenant_availability_zone ), - true <- if(node(sup) != node() and data.mode == :transaction, do: :proxy, else: true), + true <- + if(node(sup) != node() and data.mode in [:transaction, :session], + do: :proxy, + else: true + ), {:ok, opts} <- Supavisor.subscribe(sup, data.id) do Process.monitor(opts.workers.manager) data = Map.merge(data, opts.workers) @@ -951,7 +959,7 @@ defmodule Supavisor.ClientHandler do end GenServer.stop(conn, :normal, 5_000) - Logger.info("ProxyClient: Get secrets finished") + Logger.info("ClientHandler: Get secrets finished") resp end @@ -1057,4 +1065,24 @@ defmodule Supavisor.ClientHandler do idle ++ heartbeat end + + @spec app_name(any()) :: String.t() + def app_name(name) when is_binary(name), do: name + + def app_name(name) do + Logger.error("ClientHandler: Invalid application name #{inspect(name)}") + "Supavisor" + end + + @spec maybe_change_log(map()) :: atom() | nil + def maybe_change_log(%{"payload" => %{"options" => options}}) do + level = options["log_level"] && String.to_existing_atom(options["log_level"]) + + if level in [:debug, :info, :notice, :warning, :error] do + Helpers.set_log_level(level) + level + end + end + + def maybe_change_log(_), do: :ok end diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index b54191b2..cf74df61 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -70,7 +70,6 @@ defmodule Supavisor.DbHandler do mode: args.mode, replica_type: args.replica_type, reply: nil, - pool: Supavisor.get_local_pool(args.id), caller: args[:caller] || nil, client_sock: args[:client_sock] || nil, proxy: args[:proxy] || false, diff --git a/lib/supavisor/tenant_supervisor.ex b/lib/supavisor/tenant_supervisor.ex index 21d518ce..6dc4a1d9 100644 --- a/lib/supavisor/tenant_supervisor.ex +++ b/lib/supavisor/tenant_supervisor.ex @@ -5,7 +5,8 @@ defmodule Supavisor.TenantSupervisor do require Logger alias Supavisor.Manager - def start_link(%{replicas: [%{mode: :transaction} = single]} = args) do + def start_link(%{replicas: [%{mode: mode} = single]} = args) + when mode in [:transaction, :session] do {:ok, meta} = Supavisor.start_local_server(single) Logger.info("Starting ranch instance #{inspect(meta)} for #{inspect(args.id)}") name = {:via, :syn, {:tenants, args.id, meta}} From 02c43f66897e5b46f06acfceb2db4e1a74ddbe83 Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 16 Sep 2024 11:57:36 +0200 Subject: [PATCH 55/97] feat: forward errors from the database to the client (#439) --- config/config.exs | 3 +- config/test.exs | 3 +- lib/supavisor/db_handler.ex | 65 ++++++++++++++++++++++++------ lib/supavisor/protocol/server.ex | 6 +++ test/supavisor/db_handler_test.exs | 11 ++++- 5 files changed, 72 insertions(+), 16 deletions(-) diff --git a/config/config.exs b/config/config.exs index d165af37..64bc8541 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,7 +12,8 @@ config :supavisor, version: Mix.Project.config()[:version], env: Mix.env(), metrics_disabled: System.get_env("METRICS_DISABLED") == "true", - switch_active_count: System.get_env("SWITCH_ACTIVE_COUNT", "100") |> String.to_integer() + switch_active_count: System.get_env("SWITCH_ACTIVE_COUNT", "100") |> String.to_integer(), + reconnect_retries: System.get_env("RECONNECT_RETRIES", "5") |> String.to_integer() # Configures the endpoint config :supavisor, SupavisorWeb.Endpoint, diff --git a/config/test.exs b/config/test.exs index 41704b37..b33c8604 100644 --- a/config/test.exs +++ b/config/test.exs @@ -18,7 +18,8 @@ config :supavisor, metrics_blocklist: [], node_host: System.get_env("NODE_IP", "127.0.0.1"), availability_zone: System.get_env("AVAILABILITY_ZONE"), - max_pools: 3 + max_pools: 3, + reconnect_retries: System.get_env("RECONNECT_RETRIES", "5") |> String.to_integer() config :supavisor, Supavisor.Repo, username: "postgres", diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index cf74df61..b5b7c248 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -22,6 +22,7 @@ defmodule Supavisor.DbHandler do @sock_closed [:tcp_closed, :ssl_closed] @proto [:tcp, :ssl] @switch_active_count Application.compile_env(:supavisor, :switch_active_count) + @reconnect_retries Application.compile_env(:supavisor, :reconnect_retries) def start_link(config), do: :gen_statem.start_link(__MODULE__, config, hibernate_after: 5_000) @@ -73,7 +74,8 @@ defmodule Supavisor.DbHandler do caller: args[:caller] || nil, client_sock: args[:client_sock] || nil, proxy: args[:proxy] || false, - active_count: 0 + active_count: 0, + reconnect_retries: 0 } Telem.handler_action(:db_handler, :started, args.id) @@ -102,7 +104,11 @@ defmodule Supavisor.DbHandler do active: false ] - reconnect_callback = {:keep_state_and_data, {:state_timeout, @reconnect_timeout, :connect}} + maybe_reconnect_callback = fn reason -> + if data.reconnect_retries > @reconnect_retries and data.client_sock != nil, + do: {:stop, {:failed_to_connect, reason}}, + else: {:keep_state_and_data, {:state_timeout, @reconnect_timeout, :connect}} + end Telem.handler_action(:db_handler, :db_connection, data.id) @@ -121,12 +127,12 @@ defmodule Supavisor.DbHandler do {:error, reason} -> Logger.error("DbHandler: Send startup error #{inspect(reason)}") - reconnect_callback + maybe_reconnect_callback.(reason) end - {:error, error} -> - Logger.error("DbHandler: Handshake error #{inspect(error)}") - reconnect_callback + {:error, reason} -> + Logger.error("DbHandler: Handshake error #{inspect(reason)}") + maybe_reconnect_callback.(reason) end other -> @@ -134,13 +140,16 @@ defmodule Supavisor.DbHandler do "DbHandler: Connection failed #{inspect(other)} to #{inspect(auth.host)}:#{inspect(auth.port)}" ) - reconnect_callback + maybe_reconnect_callback.(other) end end - def handle_event(:state_timeout, :connect, _state, _) do - Logger.warning("DbHandler: Reconnect") - {:keep_state_and_data, {:next_event, :internal, :connect}} + def handle_event(:state_timeout, :connect, _state, data) do + retry = data.reconnect_retries + Logger.warning("DbHandler: Reconnect #{retry} to DB") + + {:keep_state, %{data | reconnect_retries: data.reconnect_retries + 1}, + {:next_event, :internal, :connect}} end def handle_event(:info, {proto, _, bin}, :authentication, data) when proto in @proto do @@ -169,12 +178,13 @@ defmodule Supavisor.DbHandler do {:keep_state, data} {:error_response, ["SFATAL", "VFATAL", "C28P01", reason, _, _, _]} -> + handle_authentication_error(data, reason) Logger.error("DbHandler: Auth error #{inspect(reason)}") {:stop, :invalid_password, data} {:error_response, error} -> Logger.error("DbHandler: Error auth response #{inspect(error)}") - {:keep_state, data} + {:stop, {:encode_and_forward, error}} {:ready_for_query, acc} -> ps = acc.ps @@ -190,7 +200,7 @@ defmodule Supavisor.DbHandler do Supavisor.set_parameter_status(data.id, ps) end - {:next_state, :idle, %{data | parameter_status: ps}, + {:next_state, :idle, %{data | parameter_status: ps, reconnect_retries: 0}, {:next_event, :internal, :check_buffer}} other -> @@ -381,6 +391,16 @@ defmodule Supavisor.DbHandler do def terminate(reason, state, data) do Telem.handler_action(:db_handler, :stopped, data.id) + if data.client_sock != nil do + message = + case reason do + {:encode_and_forward, msg} -> Server.encode_error_message(msg) + _ -> Server.error_message("XX000", inspect(reason)) + end + + HandlerHelpers.sock_send(data.client_sock, message) + end + Logger.error( "DbHandler: Terminating with reason #{inspect(reason)} when state was #{inspect(state)}" ) @@ -630,4 +650,25 @@ defmodule Supavisor.DbHandler do do: {:error_response, error} defp handle_auth_pkts(_e, acc, _data), do: acc + + @spec handle_authentication_error(map(), String.t()) :: any() + defp handle_authentication_error(%{proxy: false} = data, reason) do + tenant = Supavisor.tenant(data.id) + + for node <- [node() | Node.list()] do + :erpc.cast(node, fn -> + Cachex.del(Supavisor.Cache, {:secrets, tenant, data.user}) + Cachex.del(Supavisor.Cache, {:secrets_check, tenant, data.user}) + + Registry.dispatch(Supavisor.Registry.TenantClients, data.id, fn entries -> + for {client_handler, _meta} <- entries, + do: send(client_handler, {:disconnect, reason}) + end) + end) + end + + Supavisor.stop(data.id) + end + + defp handle_authentication_error(%{proxy: true}, _reason), do: :ok end diff --git a/lib/supavisor/protocol/server.ex b/lib/supavisor/protocol/server.ex index d1e186cd..02744356 100644 --- a/lib/supavisor/protocol/server.ex +++ b/lib/supavisor/protocol/server.ex @@ -317,6 +317,12 @@ defmodule Supavisor.Protocol.Server do [<>, message] end + @spec encode_error_message(list()) :: iodata() + def encode_error_message(message) when is_list(message) do + message = Enum.join(message, <<0>>) <> <<0, 0>> + [<>, message] + end + def decode_parameter_description("", acc), do: Enum.reverse(acc) def decode_parameter_description(<>, acc) do diff --git a/test/supavisor/db_handler_test.exs b/test/supavisor/db_handler_test.exs index f61ec753..50e314c2 100644 --- a/test/supavisor/db_handler_test.exs +++ b/test/supavisor/db_handler_test.exs @@ -14,7 +14,8 @@ defmodule Supavisor.DbHandlerTest do user: "user", mode: :transaction, replica_type: :single, - log_level: nil + log_level: nil, + reconnect_retries: 5 } {:ok, :connect, data, {_, next_event, _}} = Db.init(args) @@ -97,7 +98,13 @@ defmodule Supavisor.DbHandlerTest do ip_version: :inet } - state = Db.handle_event(:internal, nil, :connect, %{auth: auth, sock: nil, id: {"a", "b"}}) + state = + Db.handle_event(:internal, nil, :connect, %{ + auth: auth, + sock: nil, + id: {"a", "b"}, + reconnect_retries: 5 + }) assert state == {:keep_state_and_data, {:state_timeout, 2_500, :connect}} :meck.unload(:gen_tcp) From 163e4fa2269a3cf57ebf92b80d0a5c04435a4f44 Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 16 Sep 2024 11:58:37 +0200 Subject: [PATCH 56/97] feat: add search_path support on connection (#440) --- lib/supavisor.ex | 24 +++++++---- lib/supavisor/client_handler.ex | 13 ++++-- lib/supavisor/db_handler.ex | 20 +++++---- lib/supavisor/manager.ex | 2 +- lib/supavisor/monitoring/prom_ex.ex | 14 ++++-- lib/supavisor/monitoring/telem.ex | 66 +++++++++++++++++++++++------ lib/supavisor/monitoring/tenant.ex | 13 ++++-- lib/supavisor/native_handler.ex | 2 +- lib/supavisor/syn_handler.ex | 2 +- lib/supavisor/tenant_supervisor.ex | 4 +- lib/supavisor/tenants_metrics.ex | 2 +- test/integration/proxy_test.exs | 4 +- test/supavisor/db_handler_test.exs | 11 ++--- test/supavisor/prom_ex_test.exs | 2 +- test/supavisor/syn_handler_test.exs | 2 +- 15 files changed, 130 insertions(+), 51 deletions(-) diff --git a/lib/supavisor.ex b/lib/supavisor.ex index d115e830..c8827736 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -15,7 +15,7 @@ defmodule Supavisor do @type workers :: %{manager: pid, pool: pid} @type secrets :: {:password | :auth_query, fun()} @type mode :: :transaction | :session | :native | :proxy - @type id :: {{:single | :cluster, String.t()}, String.t(), mode, String.t()} + @type id :: {{:single | :cluster, String.t()}, String.t(), mode, String.t(), String.t() | nil} @type subscribe_opts :: %{workers: workers, ps: list, idle_timeout: integer} @registry Supavisor.Registry.Tenants @@ -221,8 +221,9 @@ defmodule Supavisor do end end - @spec id({:single | :cluster, String.t()}, String.t(), mode, mode, String.t()) :: id - def id(tenant, user, port_mode, user_mode, db_name) do + @spec id({:single | :cluster, String.t()}, String.t(), mode, mode, String.t(), String.t() | nil) :: + id + def id(tenant, user, port_mode, user_mode, db_name, search_path) do # temporary hack mode = if port_mode == :transaction do @@ -231,14 +232,17 @@ defmodule Supavisor do port_mode end - {tenant, user, mode, db_name} + {tenant, user, mode, db_name, search_path} end @spec tenant(id) :: String.t() - def tenant({{_, tenant}, _, _, _}), do: tenant + def tenant({{_, tenant}, _, _, _, _}), do: tenant @spec mode(id) :: atom() - def mode({_, _, mode, _}), do: mode + def mode({_, _, mode, _, _}), do: mode + + @spec search_path(id) :: String.t() | nil + def search_path({_, _, _, _, search_path}), do: search_path @spec determine_node(id, String.t() | nil) :: Node.t() def determine_node(id, availability_zone) do @@ -272,7 +276,11 @@ defmodule Supavisor do end @spec start_local_pool(id, secrets, atom()) :: {:ok, pid} | {:error, any} - def start_local_pool({{type, tenant}, _user, _mode, _db_name} = id, secrets, log_level \\ nil) do + def start_local_pool( + {{type, tenant}, _user, _mode, _db_name, _search_path} = id, + secrets, + log_level \\ nil + ) do Logger.info("Starting pool(s) for #{inspect(id)}") user = elem(secrets, 1).().alias @@ -315,7 +323,7 @@ defmodule Supavisor do defp supervisor_args( tenant_record, - {tenant, user, mode, db_name} = id, + {tenant, user, mode, db_name, _search_path} = id, {method, secrets}, log_level ) do diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index e5cbb605..04a5115f 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -171,6 +171,11 @@ defmodule Supavisor.ClientHandler do end end + def handle_event(:info, {_, _, bin}, :exchange, _) when byte_size(bin) > 1024 do + Logger.error("ClientHandler: Startup packet too large #{byte_size(bin)}") + {:stop, {:shutdown, :startup_packet_too_large}} + end + def handle_event(:info, {_, _, bin}, :exchange, data) do case Server.decode_startup_packet(bin) do {:ok, hello} -> @@ -184,7 +189,8 @@ defmodule Supavisor.ClientHandler do if user =~ rule and db_name =~ rule do log_level = maybe_change_log(hello) - event = {:hello, {type, {user, tenant_or_alias, db_name}}} + search_path = hello.payload["options"]["--search_path"] + event = {:hello, {type, {user, tenant_or_alias, db_name, search_path}}} app_name = app_name(hello.payload["application_name"]) {:keep_state, %{data | log_level: log_level, app_name: app_name}, @@ -212,7 +218,7 @@ defmodule Supavisor.ClientHandler do def handle_event( :internal, - {:hello, {type, {user, tenant_or_alias, db_name}}}, + {:hello, {type, {user, tenant_or_alias, db_name, search_path}}}, :exchange, %{sock: sock} = data ) do @@ -228,7 +234,8 @@ defmodule Supavisor.ClientHandler do user, data.mode, info.user.mode_type, - db_name + db_name, + search_path ) mode = Supavisor.mode(id) diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index b5b7c248..7b7d2c84 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -119,8 +119,9 @@ defmodule Supavisor.DbHandler do case try_ssl_handshake({:gen_tcp, sock}, auth) do {:ok, sock} -> tenant = if data.proxy, do: Supavisor.tenant(data.id) + search_path = Supavisor.search_path(data.id) - case send_startup(sock, auth, tenant) do + case send_startup(sock, auth, tenant, search_path) do :ok -> :ok = activate(sock) {:next_state, :authentication, %{data | sock: sock}} @@ -452,17 +453,20 @@ defmodule Supavisor.DbHandler do end end - @spec send_startup(Supavisor.sock(), map(), String.t() | nil) :: :ok | {:error, term} - def send_startup(sock, auth, tenant) do + @spec send_startup(Supavisor.sock(), map(), String.t() | nil, String.t() | nil) :: + :ok | {:error, term} + def send_startup(sock, auth, tenant, search_path) do user = if is_nil(tenant), do: get_user(auth), else: "#{get_user(auth)}.#{tenant}" msg = - :pgo_protocol.encode_startup_message([ - {"user", user}, - {"database", auth.database}, - {"application_name", auth.application_name} - ]) + :pgo_protocol.encode_startup_message( + [ + {"user", user}, + {"database", auth.database}, + {"application_name", auth.application_name} + ] ++ if(search_path, do: [{"options", "--search_path=#{search_path}"}], else: []) + ) sock_send(sock, msg) end diff --git a/lib/supavisor/manager.ex b/lib/supavisor/manager.ex index 55c851e4..ee86f7f8 100644 --- a/lib/supavisor/manager.ex +++ b/lib/supavisor/manager.ex @@ -39,7 +39,7 @@ defmodule Supavisor.Manager do [args | _] = Enum.filter(args.replicas, fn e -> e.replica_type == :write end) - {{type, tenant}, user, _mode, db_name} = args.id + {{type, tenant}, user, _mode, db_name, _search_path} = args.id state = %{ id: args.id, diff --git a/lib/supavisor/monitoring/prom_ex.ex b/lib/supavisor/monitoring/prom_ex.ex index c4524ce0..48b762d3 100644 --- a/lib/supavisor/monitoring/prom_ex.ex +++ b/lib/supavisor/monitoring/prom_ex.ex @@ -30,9 +30,17 @@ defmodule Supavisor.Monitoring.PromEx do end @spec remove_metrics(S.id()) :: non_neg_integer - def remove_metrics({{type, tenant}, user, mode, db_name} = id) do + def remove_metrics({{type, tenant}, user, mode, db_name, search_path} = id) do Logger.debug("Removing metrics for #{inspect(id)}") - meta = %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} + + meta = %{ + tenant: tenant, + user: user, + mode: mode, + type: type, + db_name: db_name, + search_path: search_path + } Supavisor.Monitoring.PromEx.Metrics |> :ets.select_delete([ @@ -101,7 +109,7 @@ defmodule Supavisor.Monitoring.PromEx do |> Enum.uniq() _ = - Enum.reduce(pools, metrics, fn {{_type, tenant}, _, _, _}, acc -> + Enum.reduce(pools, metrics, fn {{_type, tenant}, _, _, _, _}, acc -> {matched, rest} = Enum.split_with(acc, &String.contains?(&1, "tenant=\"#{tenant}\"")) if matched != [] do diff --git a/lib/supavisor/monitoring/telem.ex b/lib/supavisor/monitoring/telem.ex index 71f36dea..9c814511 100644 --- a/lib/supavisor/monitoring/telem.ex +++ b/lib/supavisor/monitoring/telem.ex @@ -36,12 +36,19 @@ defmodule Supavisor.Monitoring.Telem do recv_oct: recv_oct } - {{ptype, tenant}, user, mode, db_name} = id + {{ptype, tenant}, user, mode, db_name, search_path} = id :telemetry.execute( [:supavisor, type, :network, :stat], stats, - %{tenant: tenant, user: user, mode: mode, type: ptype, db_name: db_name} + %{ + tenant: tenant, + user: user, + mode: mode, + type: ptype, + db_name: db_name, + search_path: search_path + } ) {:ok, %{}} @@ -54,38 +61,66 @@ defmodule Supavisor.Monitoring.Telem do end @spec pool_checkout_time(integer(), Supavisor.id(), :local | :remote) :: :ok | nil - def pool_checkout_time(time, {{type, tenant}, user, mode, db_name}, same_box) do + def pool_checkout_time(time, {{type, tenant}, user, mode, db_name, search_path}, same_box) do telemetry_execute( [:supavisor, :pool, :checkout, :stop, same_box], %{duration: time}, - %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} + %{ + tenant: tenant, + user: user, + mode: mode, + type: type, + db_name: db_name, + search_path: search_path + } ) end @spec client_query_time(integer(), Supavisor.id()) :: :ok | nil - def client_query_time(start, {{type, tenant}, user, mode, db_name}) do + def client_query_time(start, {{type, tenant}, user, mode, db_name, search_path}) do telemetry_execute( [:supavisor, :client, :query, :stop], %{duration: System.monotonic_time() - start}, - %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} + %{ + tenant: tenant, + user: user, + mode: mode, + type: type, + db_name: db_name, + search_path: search_path + } ) end @spec client_connection_time(integer(), Supavisor.id()) :: :ok | nil - def client_connection_time(start, {{type, tenant}, user, mode, db_name}) do + def client_connection_time(start, {{type, tenant}, user, mode, db_name, search_path}) do telemetry_execute( [:supavisor, :client, :connection, :stop], %{duration: System.monotonic_time() - start}, - %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} + %{ + tenant: tenant, + user: user, + mode: mode, + type: type, + db_name: db_name, + search_path: search_path + } ) end @spec client_join(:ok | :fail, Supavisor.id() | any()) :: :ok | nil - def client_join(status, {{type, tenant}, user, mode, db_name}) do + def client_join(status, {{type, tenant}, user, mode, db_name, search_path}) do telemetry_execute( [:supavisor, :client, :joins, status], %{}, - %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} + %{ + tenant: tenant, + user: user, + mode: mode, + type: type, + db_name: db_name, + search_path: search_path + } ) end @@ -98,11 +133,18 @@ defmodule Supavisor.Monitoring.Telem do :started | :stopped | :db_connection, Supavisor.id() ) :: :ok | nil - def handler_action(handler, action, {{type, tenant}, user, mode, db_name}) do + def handler_action(handler, action, {{type, tenant}, user, mode, db_name, search_path}) do telemetry_execute( [:supavisor, handler, action, :all], %{}, - %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} + %{ + tenant: tenant, + user: user, + mode: mode, + type: type, + db_name: db_name, + search_path: search_path + } ) end diff --git a/lib/supavisor/monitoring/tenant.ex b/lib/supavisor/monitoring/tenant.ex index ff976ab8..d4fa9544 100644 --- a/lib/supavisor/monitoring/tenant.ex +++ b/lib/supavisor/monitoring/tenant.ex @@ -6,7 +6,7 @@ defmodule Supavisor.PromEx.Plugins.Tenant do alias Supavisor, as: S - @tags [:tenant, :user, :mode, :type, :db_name] + @tags [:tenant, :user, :mode, :type, :db_name, :search_path] @impl true def polling_metrics(opts) do @@ -208,11 +208,18 @@ defmodule Supavisor.PromEx.Plugins.Tenant do end @spec emit_telemetry_for_tenant({S.id(), non_neg_integer()}) :: :ok - def emit_telemetry_for_tenant({{{type, tenant}, user, mode, db_name}, count}) do + def emit_telemetry_for_tenant({{{type, tenant}, user, mode, db_name, search_path}, count}) do :telemetry.execute( [:supavisor, :connections], %{active: count}, - %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} + %{ + tenant: tenant, + user: user, + mode: mode, + type: type, + db_name: db_name, + search_path: search_path + } ) end diff --git a/lib/supavisor/native_handler.ex b/lib/supavisor/native_handler.ex index db5f68b7..d9a13da5 100644 --- a/lib/supavisor/native_handler.ex +++ b/lib/supavisor/native_handler.ex @@ -148,7 +148,7 @@ defmodule Supavisor.NativeHandler do db_name: db_name ) - id = Supavisor.id(ext_id, user, :native, :native, db_name) + id = Supavisor.id(ext_id, user, :native, :native, db_name, nil) Registry.register(Supavisor.Registry.TenantClients, id, []) payload = diff --git a/lib/supavisor/syn_handler.ex b/lib/supavisor/syn_handler.ex index 2f18bb62..49450b4e 100644 --- a/lib/supavisor/syn_handler.ex +++ b/lib/supavisor/syn_handler.ex @@ -7,7 +7,7 @@ defmodule Supavisor.SynHandler do def on_process_unregistered( :tenants, - {{_type, _tenant}, _user, _mode, _db_name} = id, + {{_type, _tenant}, _user, _mode, _db_name, _search_path} = id, _pid, meta, reason diff --git a/lib/supavisor/tenant_supervisor.ex b/lib/supavisor/tenant_supervisor.ex index 6dc4a1d9..64136049 100644 --- a/lib/supavisor/tenant_supervisor.ex +++ b/lib/supavisor/tenant_supervisor.ex @@ -35,8 +35,8 @@ defmodule Supavisor.TenantSupervisor do children = [{Manager, args} | pools] - {{type, tenant}, user, mode, db_name} = args.id - map_id = %{user: user, mode: mode, type: type, db_name: db_name} + {{type, tenant}, user, mode, db_name, search_path} = args.id + map_id = %{user: user, mode: mode, type: type, db_name: db_name, search_path: search_path} Registry.register(Supavisor.Registry.TenantSups, tenant, map_id) Supervisor.init(children, diff --git a/lib/supavisor/tenants_metrics.ex b/lib/supavisor/tenants_metrics.ex index 973dc72f..a3cf43ad 100644 --- a/lib/supavisor/tenants_metrics.ex +++ b/lib/supavisor/tenants_metrics.ex @@ -31,7 +31,7 @@ defmodule Supavisor.TenantsMetrics do active_pools = PromEx.do_cache_tenants_metrics() |> MapSet.new() MapSet.difference(state.pools, active_pools) - |> Enum.each(fn {{_type, tenant}, _, _, _} = pool -> + |> Enum.each(fn {{_type, tenant}, _, _, _, _} = pool -> Logger.debug("Removing cached metrics for #{inspect(pool)}") Cachex.del(Supavisor.Cache, {:metrics, tenant}) end) diff --git a/test/integration/proxy_test.exs b/test/integration/proxy_test.exs index 646d0e62..1df06ed6 100644 --- a/test/integration/proxy_test.exs +++ b/test/integration/proxy_test.exs @@ -196,7 +196,9 @@ defmodule Supavisor.Integration.ProxyTest do {:ok, pid} = parse_uri(url) |> single_connection() [{_, client_pid, _}] = - Supavisor.get_local_manager({{:single, @tenant}, "transaction", :transaction, "postgres"}) + Supavisor.get_local_manager( + {{:single, @tenant}, "transaction", :transaction, "postgres", nil} + ) |> :sys.get_state() |> Access.get(:tid) |> :ets.tab2list() diff --git a/test/supavisor/db_handler_test.exs b/test/supavisor/db_handler_test.exs index 50e314c2..ff53cbb5 100644 --- a/test/supavisor/db_handler_test.exs +++ b/test/supavisor/db_handler_test.exs @@ -3,11 +3,12 @@ defmodule Supavisor.DbHandlerTest do alias Supavisor.DbHandler, as: Db alias Supavisor.Protocol.Server # import Mock + @id {{:single, "tenant"}, "user", :transaction, "postgres", nil} describe "init/1" do test "starts with correct state" do args = %{ - id: {"a", "b"}, + id: @id, auth: %{}, tenant: {:single, "test_tenant"}, user_alias: "test_user_alias", @@ -58,7 +59,7 @@ defmodule Supavisor.DbHandlerTest do Db.handle_event(:internal, nil, :connect, %{ auth: auth, sock: {:gen_tcp, nil}, - id: {"a", "b"}, + id: @id, proxy: false }) @@ -76,7 +77,7 @@ defmodule Supavisor.DbHandlerTest do secrets: secrets }, sock: {:gen_tcp, :sock}, - id: {"a", "b"}, + id: @id, proxy: false }} @@ -89,7 +90,7 @@ defmodule Supavisor.DbHandlerTest do :meck.expect(:gen_tcp, :connect, fn _host, _port, _sock_opts -> {:error, "some error"} end) auth = %{ - id: {"a", "b"}, + id: @id, host: "host", port: 0, user: "some user", @@ -158,7 +159,7 @@ defmodule Supavisor.DbHandlerTest do caller_pid = self() data = %{ - id: {{:single, "tenant"}, "user", :session, "postgres"}, + id: @id, caller: caller_pid, sock: {:gen_tcp, nil}, stats: %{}, diff --git a/test/supavisor/prom_ex_test.exs b/test/supavisor/prom_ex_test.exs index 11e50b30..24e20be0 100644 --- a/test/supavisor/prom_ex_test.exs +++ b/test/supavisor/prom_ex_test.exs @@ -29,7 +29,7 @@ defmodule Supavisor.PromExTest do assert PromEx.get_metrics() =~ "tenant=\"#{@tenant}\"" :ok = GenServer.stop(proxy) - :ok = Supavisor.stop({{:single, @tenant}, user, :transaction, db_name}) + :ok = Supavisor.stop({{:single, @tenant}, user, :transaction, db_name, nil}) Process.sleep(1000) diff --git a/test/supavisor/syn_handler_test.exs b/test/supavisor/syn_handler_test.exs index 90d40657..dbb65243 100644 --- a/test/supavisor/syn_handler_test.exs +++ b/test/supavisor/syn_handler_test.exs @@ -5,7 +5,7 @@ defmodule Supavisor.SynHandlerTest do alias Ecto.Adapters.SQL.Sandbox alias Supavisor.Support.Cluster - @id {{:single, "syn_tenant"}, "postgres", :session, "postgres"} + @id {{:single, "syn_tenant"}, "postgres", :session, "postgres", nil} test "resolving conflict" do {:ok, _pid, node2} = Cluster.start_node() From 7971b255379d4cd1f5b96fed9b5fcd1a16306201 Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 16 Sep 2024 12:01:55 +0200 Subject: [PATCH 57/97] bump 2.0.1 (#441) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 227cea21..38f77a65 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0 +2.0.1 From 31aa33c48c05bede96898b7169aa6b4e537a91d0 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 18 Sep 2024 21:02:48 +0200 Subject: [PATCH 58/97] chore: silence invalid application name error (#444) --- lib/supavisor/client_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 04a5115f..0b763475 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -1077,7 +1077,7 @@ defmodule Supavisor.ClientHandler do def app_name(name) when is_binary(name), do: name def app_name(name) do - Logger.error("ClientHandler: Invalid application name #{inspect(name)}") + Logger.debug("ClientHandler: Invalid application name #{inspect(name)}") "Supavisor" end From ccf52fbf84d9a233612fdecb101ac3784cd41889 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 18 Sep 2024 21:03:57 +0200 Subject: [PATCH 59/97] fix: search_path in MetricsCleaner (#446) --- lib/supavisor/metrics_cleaner.ex | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/supavisor/metrics_cleaner.ex b/lib/supavisor/metrics_cleaner.ex index 2cb82258..9e76da9a 100644 --- a/lib/supavisor/metrics_cleaner.ex +++ b/lib/supavisor/metrics_cleaner.ex @@ -39,8 +39,17 @@ defmodule Supavisor.MetricsCleaner do tenant_registry_table = :syn_registry_by_name_tenants fn - {{_, %{type: type, mode: mode, user: user, tenant: tenant, db_name: db}} = key, _}, _ -> - case :ets.lookup(tenant_registry_table, {{type, tenant}, user, mode, db}) do + {{_, + %{ + type: type, + mode: mode, + user: user, + tenant: tenant, + db_name: db, + search_path: search_path + }} = key, _}, + _ -> + case :ets.lookup(tenant_registry_table, {{type, tenant}, user, mode, db, search_path}) do [] -> Logger.warning("Found orphaned metric: #{inspect(key)}") :ets.delete(metrics_table, key) From 006172fbb447a892a934da9b29f8e29510235692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 24 Sep 2024 13:20:56 +0200 Subject: [PATCH 60/97] chore: update mailmap (#447) Update mailmap with a Supabase email for me. --- .mailmap | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.mailmap b/.mailmap index c3543e56..84bc1c74 100644 --- a/.mailmap +++ b/.mailmap @@ -1,5 +1,8 @@ Stanislav Muzhyk + Joel Lee Joel Lee Dimitris Zorbas + +Łukasz Niemier From d579414594f6adcfb173600923d5e4200fb1c3d2 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 25 Sep 2024 11:50:28 +0200 Subject: [PATCH 61/97] chore: only supa folks in mailmap, bump version to 2.0.2 (#448) --- .mailmap | 2 -- VERSION | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.mailmap b/.mailmap index 84bc1c74..346ea615 100644 --- a/.mailmap +++ b/.mailmap @@ -3,6 +3,4 @@ Stanislav Muzhyk Joel Lee Joel Lee -Dimitris Zorbas - Łukasz Niemier diff --git a/VERSION b/VERSION index 38f77a65..e9307ca5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.1 +2.0.2 From d4d862a7647abbf3df8e51266ba6ca3000cb6808 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 25 Sep 2024 15:18:56 +0200 Subject: [PATCH 62/97] fix: also notify the client about ready_for_query in session mode (#450) --- lib/supavisor/db_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 7b7d2c84..31047f7f 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -298,7 +298,7 @@ defmodule Supavisor.DbHandler do {_, stats} = Telem.network_usage(:db, data.sock, data.id, data.stats) data = - if data.mode == :transaction do + if data.mode in [:transaction, :session] do ClientHandler.db_status(data.caller, :ready_for_query, bin) %{data | stats: stats, caller: nil, client_sock: nil, active_count: 0} else From 916331faca60e8e33728ceec254d4e4859ceb2ff Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 30 Sep 2024 12:33:23 +0200 Subject: [PATCH 63/97] fix: handling of active_count (#451) --- VERSION | 2 +- config/test.exs | 2 +- lib/supavisor/client_handler.ex | 46 ++++++++----- lib/supavisor/db_handler.ex | 4 +- test/integration/proxy_test.exs | 114 ++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 59 deletions(-) diff --git a/VERSION b/VERSION index e9307ca5..50ffc5aa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.2 +2.0.3 diff --git a/config/test.exs b/config/test.exs index b33c8604..e1019ca1 100644 --- a/config/test.exs +++ b/config/test.exs @@ -18,7 +18,7 @@ config :supavisor, metrics_blocklist: [], node_host: System.get_env("NODE_IP", "127.0.0.1"), availability_zone: System.get_env("AVAILABILITY_ZONE"), - max_pools: 3, + max_pools: 5, reconnect_retries: System.get_env("RECONNECT_RETRIES", "5") |> String.to_integer() config :supavisor, Supavisor.Repo, diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 0b763475..9e4f9bbc 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -84,7 +84,6 @@ defmodule Supavisor.ClientHandler do heartbeat_interval: 0, connection_start: System.monotonic_time(), log_level: nil, - db_sock: nil, auth: %{}, tenant_availability_zone: nil, local: opts[:local] || false, @@ -506,25 +505,32 @@ defmodule Supavisor.ClientHandler do end # handle Sync message - def handle_event(:info, {proto, _, <>}, :idle, data) + def handle_event(:info, {proto, _, <> = msg}, :idle, data) when proto in @proto do - Logger.info("ClientHandler: Receive sync") - :ok = HandlerHelpers.sock_send(data.sock, Server.ready_for_query()) - {:keep_state_and_data, handle_actions(data)} + Logger.debug("ClientHandler: Receive sync") + + # db_pid can be nil in transaction mode, so we will send ready_for_query + # without checking out a direct connection. If there is a linked db_pid, + # we will forward the message to it + if data.db_pid != nil, + do: :ok = sock_send_maybe_active_once(msg, data), + else: :ok = HandlerHelpers.sock_send(data.sock, Server.ready_for_query()) + + {:keep_state, %{data | active_count: 0}, handle_actions(data)} end def handle_event(:info, {proto, _, <> = msg}, _, data) when proto in @proto do - Logger.warning("ClientHandler: Receive sync while not idle") - :ok = HandlerHelpers.sock_send(elem(data.db_pid, 2), msg) - :keep_state_and_data + Logger.debug("ClientHandler: Receive sync while not idle") + :ok = sock_send_maybe_active_once(msg, data) + {:keep_state, %{data | active_count: 0}, handle_actions(data)} end def handle_event(:info, {proto, _, <> = msg}, _, data) when proto in @proto do - Logger.warning("ClientHandler: Receive flush while not idle") - :ok = HandlerHelpers.sock_send(elem(data.db_pid, 2), msg) - :keep_state_and_data + Logger.debug("ClientHandler: Receive flush while not idle") + :ok = sock_send_maybe_active_once(msg, data) + {:keep_state, %{data | active_count: 0}, handle_actions(data)} end # incoming query with a single pool @@ -574,10 +580,7 @@ defmodule Supavisor.ClientHandler do def handle_event(_, {proto, _, bin}, :busy, data) when proto in @proto do Logger.debug("ClientHandler: Forward query to db #{inspect(bin)} #{inspect(data.db_pid)}") - if data.active_count > @switch_active_count, - do: HandlerHelpers.active_once(data.sock) - - case HandlerHelpers.sock_send(elem(data.db_pid, 2), bin) do + case sock_send_maybe_active_once(bin, data) do :ok -> {:keep_state, %{data | active_count: data.active_count + 1}} @@ -1092,4 +1095,17 @@ defmodule Supavisor.ClientHandler do end def maybe_change_log(_), do: :ok + + @spec sock_send_maybe_active_once(binary(), map()) :: :ok | {:error, term()} + def sock_send_maybe_active_once(bin, data) do + Logger.debug("ClientHandler: Send maybe active once") + active_count = data.active_count + + if active_count > @switch_active_count do + Logger.debug("ClientHandler: Activate socket #{inspect(active_count)}") + HandlerHelpers.active_once(data.sock) + end + + HandlerHelpers.sock_send(elem(data.db_pid, 2), bin) + end end diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 31047f7f..802fc645 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -297,8 +297,10 @@ defmodule Supavisor.DbHandler do if String.ends_with?(bin, Server.ready_for_query()) do {_, stats} = Telem.network_usage(:db, data.sock, data.id, data.stats) + # in transaction mode, we need to notify the client when the transaction is finished, + # after which it will unlink the direct db connection process from itself. data = - if data.mode in [:transaction, :session] do + if data.mode == :transaction do ClientHandler.db_status(data.caller, :ready_for_query, bin) %{data | stats: stats, caller: nil, client_sock: nil, active_count: 0} else diff --git a/test/integration/proxy_test.exs b/test/integration/proxy_test.exs index 1df06ed6..cd9b9619 100644 --- a/test/integration/proxy_test.exs +++ b/test/integration/proxy_test.exs @@ -283,58 +283,90 @@ defmodule Supavisor.Integration.ProxyTest do }} = parse_uri(url) |> single_connection() end - test "max_pools limit" do + # test "max_pools limit" do + # Process.flag(:trap_exit, true) + # db_conf = Application.get_env(:supavisor, Repo) + # port = Application.get_env(:supavisor, :proxy_port_transaction) + + # tenant = "max_pool_tenant" + + # {:ok, pid1} = + # Keyword.merge(db_conf, + # username: "postgres.#{tenant}", + # port: port + # ) + # |> single_connection() + + # assert Supavisor.count_pools(tenant) == 1 + + # {:ok, pid2} = + # Keyword.merge(db_conf, + # username: "session.#{tenant}", + # port: port + # ) + # |> single_connection() + + # assert Supavisor.count_pools(tenant) == 2 + + # {:ok, pid3} = + # Keyword.merge(db_conf, + # username: "transaction.#{tenant}", + # port: port + # ) + # |> single_connection() + + # assert Supavisor.count_pools(tenant) == 3 + + # connection_opts = + # Keyword.merge(db_conf, + # username: "max_clients.#{tenant}", + # port: port + # ) + + # assert {:error, + # %Postgrex.Error{ + # postgres: %{ + # code: :internal_error, + # message: "Max pools count reached", + # unknown: "FATAL", + # severity: "FATAL", + # pg_code: "XX000" + # } + # }} = single_connection(connection_opts) + + # for pid <- [pid1, pid2, pid3], do: :gen_statem.stop(pid) + # end + + test "active_count doesn't block" do Process.flag(:trap_exit, true) db_conf = Application.get_env(:supavisor, Repo) - port = Application.get_env(:supavisor, :proxy_port_transaction) - - tenant = "max_pool_tenant" - - {:ok, pid1} = - Keyword.merge(db_conf, - username: "postgres.#{tenant}", - port: port - ) - |> single_connection() - - assert Supavisor.count_pools(tenant) == 1 + port = Application.get_env(:supavisor, :proxy_port_session) - {:ok, pid2} = + connection_opts = Keyword.merge(db_conf, - username: "session.#{tenant}", + username: db_conf[:username] <> "." <> @tenant, port: port ) - |> single_connection() - assert Supavisor.count_pools(tenant) == 2 + assert {:ok, pid} = single_connection(connection_opts) - {:ok, pid3} = - Keyword.merge(db_conf, - username: "transaction.#{tenant}", - port: port - ) - |> single_connection() + id = {{:single, @tenant}, db_conf[:username], :session, db_conf[:database], nil} + [{client_pid, _}] = Registry.lookup(Supavisor.Registry.TenantClients, id) - assert Supavisor.count_pools(tenant) == 3 + assert match?({_, %{active_count: 1}}, :sys.get_state(client_pid)) - connection_opts = - Keyword.merge(db_conf, - username: "max_clients.#{tenant}", - port: port - ) - - assert {:error, - %Postgrex.Error{ - postgres: %{ - code: :internal_error, - message: "Max pools count reached", - unknown: "FATAL", - severity: "FATAL", - pg_code: "XX000" - } - }} = single_connection(connection_opts) + Enum.each(0..200, fn _ -> + P.SimpleConnection.call(pid, {:query, "select 1;"}) + end) - for pid <- [pid1, pid2, pid3], do: :gen_statem.stop(pid) + assert match?( + [ + %Postgrex.Result{ + command: :select + } + ], + P.SimpleConnection.call(pid, {:query, "select 1;"}) + ) end defp single_connection(db_conf, c_port \\ nil) when is_list(db_conf) do From 24c54e311b8ef11792382e041c8c60c4ff8d3df1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Thu, 3 Oct 2024 15:01:32 +0200 Subject: [PATCH 64/97] chore: set service description filetype to systemd (#454) --- deploy/service/supavisor.service | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/service/supavisor.service b/deploy/service/supavisor.service index 4a3f362e..25dc545d 100644 --- a/deploy/service/supavisor.service +++ b/deploy/service/supavisor.service @@ -19,3 +19,5 @@ WantedBy=multi-user.target [Service] TasksMax=infinity + +# vi: ft=systemd From 14d599bdcb4643274f1ad589da33035a4e0face8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Thu, 3 Oct 2024 16:11:41 +0200 Subject: [PATCH 65/97] chore: switch to upstream PromEx (#453) --- mix.exs | 3 +-- mix.lock | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index d4636aac..4f3bc45f 100644 --- a/mix.exs +++ b/mix.exs @@ -56,8 +56,7 @@ defmodule Supavisor.MixProject do {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, {:benchee, "~> 1.3", only: :dev}, - # TODO: point it to Supabase fork of prom_ex when available - {:prom_ex, github: "hauleth/prom_ex", branch: "ft/add-peep-storage"}, + {:prom_ex, "~> 1.10"}, {:open_api_spex, "~> 3.16"}, {:libcluster, "~> 3.3.1"}, {:logflare_logger_backend, github: "Logflare/logflare_logger_backend", tag: "v0.11.4"}, diff --git a/mix.lock b/mix.lock index d44d6015..bd1d50d1 100644 --- a/mix.lock +++ b/mix.lock @@ -61,7 +61,7 @@ "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "poolboy": {:git, "https://github.com/abc3/poolboy.git", "999ec7f5c7282d515020bb058b4832029d6d07bc", [tag: "v0.0.2"]}, "postgrex": {:hex, :postgrex, "0.19.0", "f7d50e50cb42e0a185f5b9a6095125a9ab7e4abccfbe2ab820ab9aa92b71dbab", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "dba2d2a0a8637defbf2307e8629cb2526388ba7348f67d04ec77a5d6a72ecfae"}, - "prom_ex": {:git, "https://github.com/hauleth/prom_ex.git", "d6b41ffc6ba77a1dbbbf601eaa76af15a066c101", [branch: "ft/add-peep-storage"]}, + "prom_ex": {:hex, :prom_ex, "1.10.0", "2fe14cd0b2f7f8688280b02861c58da9624571f82a878b203825fb2bec206195", [:mix], [{:absinthe, ">= 1.7.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.1.0", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.11.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.18", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.10.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.4", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:peep, "~> 3.0", [hex: :peep, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.7.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.20.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.16.0", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 2.6.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.2", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.1", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "55d6ca87519b14202570dda5f29bf3f52e14b940eaf2ddcd89ad41347fa5781a"}, "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, "recon": {:hex, :recon, "2.5.5", "c108a4c406fa301a529151a3bb53158cadc4064ec0c5f99b03ddb8c0e4281bdf", [:mix, :rebar3], [], "hexpm", "632a6f447df7ccc1a4a10bdcfce71514412b16660fe59deca0fcf0aa3c054404"}, "req": {:hex, :req, "0.5.4", "e375e4812adf83ffcf787871d7a124d873e983e3b77466e6608b973582f7f837", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "a17998ffe2ef54f79bfdd782ef9f4cbf987d93851e89444cbc466a6a25eee494"}, @@ -72,6 +72,7 @@ "syn": {:hex, :syn, "3.3.0", "4684a909efdfea35ce75a9662fc523e4a8a4e8169a3df275e4de4fa63f99c486", [:rebar3], [], "hexpm", "e58ee447bc1094bdd21bf0acc102b1fbf99541a508cd48060bf783c245eaf7d6"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"}, + "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"}, "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, "tesla": {:hex, :tesla, "1.11.2", "24707ac48b52f72f88fc05d242b1c59a85d1ee6f16f19c312d7d3419665c9cd5", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "c549cd03aec6a7196a641689dd378b799e635eb393f689b4bd756f750c7a4014"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, From 8cf032e8b80e4a3ce291bac3adb592219067de3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Fri, 4 Oct 2024 09:12:03 +0200 Subject: [PATCH 66/97] chore: add middle name to mailmap (#455) --- .mailmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index 346ea615..4bc24c8f 100644 --- a/.mailmap +++ b/.mailmap @@ -3,4 +3,4 @@ Stanislav Muzhyk Joel Lee Joel Lee -Łukasz Niemier +Łukasz Jan Niemier From 4f6ed069d01e62db168bba3731f2a71528d2a1ee Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 4 Oct 2024 18:56:15 +0200 Subject: [PATCH 67/97] fix: add local proxy multiplier (#456) --- VERSION | 2 +- config/runtime.exs | 3 ++- config/test.exs | 3 ++- lib/supavisor.ex | 2 +- test/supavisor/prom_ex_test.exs | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index 50ffc5aa..2165f8f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.3 +2.0.4 diff --git a/config/runtime.exs b/config/runtime.exs index 531f1cf4..4bcec40b 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -160,7 +160,8 @@ if config_env() != :test do api_blocklist: System.get_env("API_TOKEN_BLOCKLIST", "") |> String.split(","), metrics_blocklist: System.get_env("METRICS_TOKEN_BLOCKLIST", "") |> String.split(","), node_host: System.get_env("NODE_IP", "127.0.0.1"), - availability_zone: System.get_env("AVAILABILITY_ZONE") + availability_zone: System.get_env("AVAILABILITY_ZONE"), + local_proxy_multiplier: System.get_env("LOCAL_PROXY_MULTIPLIER", "20") |> String.to_integer() config :supavisor, Supavisor.Repo, url: System.get_env("DATABASE_URL", "ecto://postgres:postgres@localhost:6432/postgres"), diff --git a/config/test.exs b/config/test.exs index e1019ca1..0b6d37a1 100644 --- a/config/test.exs +++ b/config/test.exs @@ -19,7 +19,8 @@ config :supavisor, node_host: System.get_env("NODE_IP", "127.0.0.1"), availability_zone: System.get_env("AVAILABILITY_ZONE"), max_pools: 5, - reconnect_retries: System.get_env("RECONNECT_RETRIES", "5") |> String.to_integer() + reconnect_retries: System.get_env("RECONNECT_RETRIES", "5") |> String.to_integer(), + local_proxy_multiplier: System.get_env("LOCAL_PROXY_MULTIPLIER", "20") |> String.to_integer() config :supavisor, Supavisor.Repo, username: "postgres", diff --git a/lib/supavisor.ex b/lib/supavisor.ex index c8827736..2ff55f1c 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -414,7 +414,7 @@ defmodule Supavisor do else: {1, 100} opts = %{ - max_connections: max_clients, + max_connections: max_clients * Application.get_env(:supavisor, :local_proxy_multiplier), num_acceptors: max(acceptors, 10), socket_opts: [port: 0, keepalive: true] } diff --git a/test/supavisor/prom_ex_test.exs b/test/supavisor/prom_ex_test.exs index 24e20be0..df684260 100644 --- a/test/supavisor/prom_ex_test.exs +++ b/test/supavisor/prom_ex_test.exs @@ -31,7 +31,7 @@ defmodule Supavisor.PromExTest do :ok = GenServer.stop(proxy) :ok = Supavisor.stop({{:single, @tenant}, user, :transaction, db_name, nil}) - Process.sleep(1000) + Process.sleep(3000) refute PromEx.get_metrics() =~ "tenant=\"#{@tenant}\"" end From ff20689ff0e808865a217b2770c6e373503f2fc1 Mon Sep 17 00:00:00 2001 From: Stas Date: Tue, 8 Oct 2024 13:58:00 +0200 Subject: [PATCH 68/97] fix: subscription retries (#461) --- config/config.exs | 3 +- config/test.exs | 1 + lib/supavisor/client_handler.ex | 51 +++++++++++++++++++++++---------- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/config/config.exs b/config/config.exs index 64bc8541..36ca234d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -13,7 +13,8 @@ config :supavisor, env: Mix.env(), metrics_disabled: System.get_env("METRICS_DISABLED") == "true", switch_active_count: System.get_env("SWITCH_ACTIVE_COUNT", "100") |> String.to_integer(), - reconnect_retries: System.get_env("RECONNECT_RETRIES", "5") |> String.to_integer() + reconnect_retries: System.get_env("RECONNECT_RETRIES", "5") |> String.to_integer(), + subscribe_retries: System.get_env("SUBSCRIBE_RETRIES", "20") |> String.to_integer() # Configures the endpoint config :supavisor, SupavisorWeb.Endpoint, diff --git a/config/test.exs b/config/test.exs index 0b6d37a1..0afa2fab 100644 --- a/config/test.exs +++ b/config/test.exs @@ -20,6 +20,7 @@ config :supavisor, availability_zone: System.get_env("AVAILABILITY_ZONE"), max_pools: 5, reconnect_retries: System.get_env("RECONNECT_RETRIES", "5") |> String.to_integer(), + subscribe_retries: System.get_env("SUBSCRIBE_RETRIES", "5") |> String.to_integer(), local_proxy_multiplier: System.get_env("LOCAL_PROXY_MULTIPLIER", "20") |> String.to_integer() config :supavisor, Supavisor.Repo, diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 9e4f9bbc..c732a13d 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -13,6 +13,8 @@ defmodule Supavisor.ClientHandler do @proto [:tcp, :ssl] @cancel_query_msg <<16::32, 1234::16, 5678::16>> @switch_active_count Application.compile_env(:supavisor, :switch_active_count) + @subscribe_retries Application.compile_env(:supavisor, :subscribe_retries) + @timeout_subscribe 500 alias Supavisor.{ DbHandler, @@ -89,7 +91,8 @@ defmodule Supavisor.ClientHandler do local: opts[:local] || false, active_count: 0, peer_ip: Helpers.peer_ip(sock), - app_name: nil + app_name: nil, + subscribe_retries: 0 } :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) @@ -424,23 +427,28 @@ defmodule Supavisor.ClientHandler do {:stop, {:shutdown, :max_pools_reached}} :proxy -> - {:ok, %{port: port, host: host}} = Supavisor.get_pool_ranch(data.id) - - auth = - Map.merge(data.auth, %{ - port: port, - host: to_charlist(host), - ip_version: :inet, - upstream_ssl: false, - upstream_tls_ca: nil, - upstream_verify: nil - }) - - {:keep_state, %{data | auth: auth}, {:next_event, :internal, :connect_db}} + case Supavisor.get_pool_ranch(data.id) do + {:ok, %{port: port, host: host}} -> + auth = + Map.merge(data.auth, %{ + port: port, + host: to_charlist(host), + ip_version: :inet, + upstream_ssl: false, + upstream_tls_ca: nil, + upstream_verify: nil + }) + + {:keep_state, %{data | auth: auth}, {:next_event, :internal, :connect_db}} + + other -> + Logger.error("ClientHandler: Subscribe proxy error: #{inspect(other)}") + timeout_subscribe_or_terminate(data) + end error -> Logger.error("ClientHandler: Subscribe error: #{inspect(error)}") - {:keep_state_and_data, {:timeout, 1000, :subscribe}} + timeout_subscribe_or_terminate(data) end end @@ -1108,4 +1116,17 @@ defmodule Supavisor.ClientHandler do HandlerHelpers.sock_send(elem(data.db_pid, 2), bin) end + + @spec timeout_subscribe_or_terminate(map()) :: :gen_statem.handle_event_result() + def timeout_subscribe_or_terminate(%{subscribe_retries: subscribe_retries} = data) do + if subscribe_retries < @subscribe_retries do + Logger.warning("ClientHandler: Retry subscribe") + + {:keep_state, %{data | subscribe_retries: subscribe_retries + 1}, + {:timeout, @timeout_subscribe, :subscribe}} + else + Logger.error("ClientHandler: Terminate after retries") + {:stop, {:shutdown, :subscribe_retries}} + end + end end From 26abc2c3ed670af1a0e8dd1ab911c26ce87885e8 Mon Sep 17 00:00:00 2001 From: Stas Date: Tue, 8 Oct 2024 16:54:17 +0200 Subject: [PATCH 69/97] feat: shorter retry connect timeout for proxy mode --- VERSION | 2 +- lib/supavisor/client_handler.ex | 2 +- lib/supavisor/db_handler.ex | 15 +++++++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/VERSION b/VERSION index 2165f8f9..e0102586 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.4 +2.0.5 diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index c732a13d..f9557bf3 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -1120,7 +1120,7 @@ defmodule Supavisor.ClientHandler do @spec timeout_subscribe_or_terminate(map()) :: :gen_statem.handle_event_result() def timeout_subscribe_or_terminate(%{subscribe_retries: subscribe_retries} = data) do if subscribe_retries < @subscribe_retries do - Logger.warning("ClientHandler: Retry subscribe") + Logger.warning("ClientHandler: Retry subscribe #{inspect(subscribe_retries)}") {:keep_state, %{data | subscribe_retries: subscribe_retries + 1}, {:timeout, @timeout_subscribe, :subscribe}} diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 802fc645..6e0d5c51 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -19,6 +19,7 @@ defmodule Supavisor.DbHandler do @type state :: :connect | :authentication | :idle | :busy @reconnect_timeout 2_500 + @reconnect_timeout_proxy 500 @sock_closed [:tcp_closed, :ssl_closed] @proto [:tcp, :ssl] @switch_active_count Application.compile_env(:supavisor, :switch_active_count) @@ -107,7 +108,7 @@ defmodule Supavisor.DbHandler do maybe_reconnect_callback = fn reason -> if data.reconnect_retries > @reconnect_retries and data.client_sock != nil, do: {:stop, {:failed_to_connect, reason}}, - else: {:keep_state_and_data, {:state_timeout, @reconnect_timeout, :connect}} + else: {:keep_state_and_data, {:state_timeout, reconnect_timeout(data), :connect}} end Telem.handler_action(:db_handler, :db_connection, data.id) @@ -149,8 +150,7 @@ defmodule Supavisor.DbHandler do retry = data.reconnect_retries Logger.warning("DbHandler: Reconnect #{retry} to DB") - {:keep_state, %{data | reconnect_retries: data.reconnect_retries + 1}, - {:next_event, :internal, :connect}} + {:keep_state, %{data | reconnect_retries: retry + 1}, {:next_event, :internal, :connect}} end def handle_event(:info, {proto, _, bin}, :authentication, data) when proto in @proto do @@ -347,7 +347,7 @@ defmodule Supavisor.DbHandler do Logger.error("DbHandler: Connection closed when state was #{state}") if Application.get_env(:supavisor, :reconnect_on_db_close), - do: {:next_state, :connect, data, {:state_timeout, @reconnect_timeout, :connect}}, + do: {:next_state, :connect, data, {:state_timeout, reconnect_timeout(data), :connect}}, else: {:stop, :db_termination, data} end @@ -677,4 +677,11 @@ defmodule Supavisor.DbHandler do end defp handle_authentication_error(%{proxy: true}, _reason), do: :ok + + @spec reconnect_timeout(map()) :: pos_integer() + def reconnect_timeout(%{proxy: true}), + do: @reconnect_timeout_proxy + + def reconnect_timeout(_), + do: @reconnect_timeout end From 5759ec63486e366a02ff53d1d996857bfe9045db Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 11 Oct 2024 20:41:00 +0200 Subject: [PATCH 70/97] cache for the pool conf --- lib/supavisor.ex | 4 +++- lib/supavisor/tenants.ex | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 2ff55f1c..b7c355d0 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -152,6 +152,7 @@ defmodule Supavisor do {:secrets, ^tenant, ^user} = key, acc -> del.(key, acc) {:user_cache, _, ^user, ^tenant, _} = key, acc -> del.(key, acc) {:tenant_cache, ^tenant, _} = key, acc -> del.(key, acc) + {:pool_config_cache, ^tenant, ^user} = key, acc -> del.(key, acc) _, acc -> acc end) end @@ -173,6 +174,7 @@ defmodule Supavisor do {:secrets, ^tenant, _} -> del.(key, acc) {:user_cache, _, _, ^tenant, _} -> del.(key, acc) {:tenant_cache, ^tenant, _} -> del.(key, acc) + {:pool_config_cache, ^tenant, _} -> del.(key, acc) _ -> acc end @@ -286,7 +288,7 @@ defmodule Supavisor do user = elem(secrets, 1).().alias case type do - :single -> Tenants.get_pool_config(tenant, user) + :single -> Tenants.get_pool_config_cache(tenant, user) :cluster -> Tenants.get_cluster_config(tenant, user) end |> case do diff --git a/lib/supavisor/tenants.ex b/lib/supavisor/tenants.ex index d8ca5545..7f039767 100644 --- a/lib/supavisor/tenants.ex +++ b/lib/supavisor/tenants.ex @@ -141,6 +141,18 @@ defmodule Supavisor.Tenants do ) end + def get_pool_config_cache(external_id, user, ttl \\ nil) do + ttl = if is_nil(ttl), do: :timer.hours(24), else: ttl + cache_key = {:pool_config_cache, external_id, user} + + case Cachex.fetch(Supavisor.Cache, cache_key, fn _key -> + {:commit, {:cached, get_pool_config(external_id, user)}, ttl: ttl} + end) do + {_, {:cached, value}} -> value + {_, {:cached, value}, _} -> value + end + end + @spec get_cluster_config(String.t(), String.t()) :: [ClusterTenants.t()] | {:error, any()} def get_cluster_config(external_id, user) do case Repo.all(ClusterTenants, cluster_alias: external_id) do From 4b7f7fe73ab64f083890e4624c441c545cfa5818 Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 14 Oct 2024 16:30:19 +0200 Subject: [PATCH 71/97] update def value --- lib/supavisor/tenants.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/supavisor/tenants.ex b/lib/supavisor/tenants.ex index 7f039767..f8a8e05c 100644 --- a/lib/supavisor/tenants.ex +++ b/lib/supavisor/tenants.ex @@ -141,7 +141,7 @@ defmodule Supavisor.Tenants do ) end - def get_pool_config_cache(external_id, user, ttl \\ nil) do + def get_pool_config_cache(external_id, user, ttl \\ :timer.hours(24)) do ttl = if is_nil(ttl), do: :timer.hours(24), else: ttl cache_key = {:pool_config_cache, external_id, user} From de98cd65ea43cf7e7a3e3d4da005081ff6785bc5 Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 25 Oct 2024 14:10:51 +0200 Subject: [PATCH 72/97] feat: periodically check auth_query creds (#464) --- lib/supavisor.ex | 4 + lib/supavisor/application.ex | 5 +- lib/supavisor/client_handler.ex | 1 - lib/supavisor/helpers.ex | 1 + lib/supavisor/secret_checker.ex | 116 +++++++++++++++++++++++++++++ lib/supavisor/tenant_supervisor.ex | 3 +- 6 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 lib/supavisor/secret_checker.ex diff --git a/lib/supavisor.ex b/lib/supavisor.ex index b7c355d0..5507ccad 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -333,6 +333,7 @@ defmodule Supavisor do db_host: db_host, db_port: db_port, db_database: db_database, + auth_query: auth_query, default_parameter_status: ps, ip_version: ip_ver, default_pool_size: def_pool_size, @@ -345,6 +346,7 @@ defmodule Supavisor do db_user: db_user, db_password: db_pass, pool_size: pool_size, + db_user_alias: alias, # mode_type: mode_type, max_clients: max_clients } @@ -363,6 +365,8 @@ defmodule Supavisor do sni_hostname: if(sni_hostname != nil, do: to_charlist(sni_hostname)), port: db_port, user: db_user, + alias: alias, + auth_query: auth_query, database: if(db_name != nil, do: db_name, else: db_database), password: fn -> db_pass end, application_name: "Supavisor", diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index 85c7a462..2b7321b1 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -77,7 +77,10 @@ defmodule Supavisor.Application do {Registry, keys: :unique, name: Supavisor.Registry.ManagerTables}, {Registry, keys: :unique, name: Supavisor.Registry.PoolPids}, {Registry, keys: :duplicate, name: Supavisor.Registry.TenantSups}, - {Registry, keys: :duplicate, name: Supavisor.Registry.TenantClients}, + {Registry, + keys: :duplicate, + name: Supavisor.Registry.TenantClients, + partitions: System.schedulers_online()}, {Cluster.Supervisor, [topologies, [name: Supavisor.ClusterSupervisor]]}, Supavisor.Repo, # Start the Telemetry supervisor diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index f9557bf3..557dbd4f 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -883,7 +883,6 @@ defmodule Supavisor.ClientHandler do host: to_charlist(info.tenant.db_host), sni_hostname: if(info.tenant.sni_hostname != nil, do: to_charlist(info.tenant.sni_hostname)), - ip_version: Helpers.ip_version(info.tenant.ip_version, info.tenant.db_host), port: info.tenant.db_port, user: user, password: info.user.db_password, diff --git a/lib/supavisor/helpers.ex b/lib/supavisor/helpers.ex index 0177836b..dc8bf8e4 100644 --- a/lib/supavisor/helpers.ex +++ b/lib/supavisor/helpers.ex @@ -201,6 +201,7 @@ defmodule Supavisor.Helpers do """ @spec detect_ip_version(String.t()) :: :inet | :inet6 def detect_ip_version(host) when is_binary(host) do + Logger.info("Detecting IP version for #{host}") host = String.to_charlist(host) case :inet.gethostbyname(host) do diff --git a/lib/supavisor/secret_checker.ex b/lib/supavisor/secret_checker.ex new file mode 100644 index 00000000..3e02043b --- /dev/null +++ b/lib/supavisor/secret_checker.ex @@ -0,0 +1,116 @@ +defmodule Supavisor.SecretChecker do + @moduledoc false + + use GenServer + require Logger + + alias Supavisor.Helpers + + @interval :timer.seconds(15) + + def start_link(args) do + name = {:via, Registry, {Supavisor.Registry.Tenants, {:secret_checker, args.id}}} + + GenServer.start_link(__MODULE__, args, name: name) + end + + def init(args) do + Logger.debug("SecretChecker: Starting secret checker") + tenant = Supavisor.tenant(args.id) + + %{auth: auth, user: user} = Enum.find(args.replicas, fn e -> e.replica_type == :write end) + + state = %{ + tenant: tenant, + auth: auth, + user: user, + key: {:secrets, tenant, user}, + ttl: args[:ttl] || :timer.hours(24), + conn: nil, + check_ref: check() + } + + Logger.metadata(project: tenant, user: user) + {:ok, state, {:continue, :init_conn}} + end + + def handle_continue(:init_conn, %{auth: auth} = state) do + ssl_opts = + if auth.upstream_ssl and auth.upstream_verify == "peer" do + [ + {:verify, :verify_peer}, + {:cacerts, [Helpers.upstream_cert(auth.upstream_tls_ca)]}, + {:server_name_indication, auth.host}, + {:customize_hostname_check, [{:match_fun, fn _, _ -> true end}]} + ] + end + + {:ok, conn} = + Postgrex.start_link( + hostname: auth.host, + port: auth.port, + database: auth.database, + password: auth.password.(), + username: auth.user, + parameters: [application_name: "Supavisor auth_query"], + ssl: auth.upstream_ssl, + socket_options: [ + auth.ip_version + ], + queue_target: 1_000, + queue_interval: 5_000, + ssl_opts: ssl_opts || [] + ) + + # kill the postgrex connection if the current process exits unexpectedly + Process.link(conn) + {:noreply, %{state | conn: conn}} + end + + def handle_info(:check, state) do + Logger.debug("Checking secrets") + check_secrets(state) + Logger.debug("Secrets checked") + {:noreply, %{state | check_ref: check()}} + end + + def handle_info(msg, state) do + Logger.error("Unexpected message: #{inspect(msg)}") + {:noreply, state} + end + + def terminate(_, state) do + :gen_statem.stop(state.conn) + :ok + end + + def check(interval \\ @interval), + do: Process.send_after(self(), :check, interval) + + def check_secrets(%{auth: auth, user: user, conn: conn} = state) do + case Helpers.get_user_secret(conn, auth.auth_query, user) do + {:ok, secret} -> + method = if secret.digest == :md5, do: :auth_query_md5, else: :auth_query + secrets = Map.put(secret, :alias, auth.alias) + + update_cache = + case Cachex.get(Supavisor.Cache, state.key) do + {:ok, {:cached, {_, {old_method, old_secrets}}}} -> + method != old_method or secrets != old_secrets.() + + other -> + Logger.error("Failed to get cache: #{inspect(other)}") + true + end + + if update_cache do + Logger.info("Secrets changed or not present, updating cache") + value = {:ok, {method, fn -> secrets end}} + Cachex.put(Supavisor.Cache, state.key, {:cached, value}, expire: :timer.hours(24)) + end + + other -> + Logger.error("Failed to get secret: #{inspect(other)}") + end + end +end diff --git a/lib/supavisor/tenant_supervisor.ex b/lib/supavisor/tenant_supervisor.ex index 64136049..9a8739aa 100644 --- a/lib/supavisor/tenant_supervisor.ex +++ b/lib/supavisor/tenant_supervisor.ex @@ -4,6 +4,7 @@ defmodule Supavisor.TenantSupervisor do require Logger alias Supavisor.Manager + alias Supavisor.SecretChecker def start_link(%{replicas: [%{mode: mode} = single]} = args) when mode in [:transaction, :session] do @@ -33,7 +34,7 @@ defmodule Supavisor.TenantSupervisor do } end) - children = [{Manager, args} | pools] + children = [{Manager, args}, {SecretChecker, args} | pools] {{type, tenant}, user, mode, db_name, search_path} = args.id map_id = %{user: user, mode: mode, type: type, db_name: db_name, search_path: search_path} From 664fc5cfa1dd1c6e2c9845e2340dc6be95f7f509 Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 28 Oct 2024 15:18:19 +0100 Subject: [PATCH 73/97] chore: name validation for user and db_name (#465) --- lib/supavisor/client_handler.ex | 7 +---- lib/supavisor/helpers.ex | 8 ++++++ test/supavisor/helpers_test.exs | 48 +++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 557dbd4f..31a70ea2 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -184,12 +184,7 @@ defmodule Supavisor.ClientHandler do Logger.debug("ClientHandler: Client startup message: #{inspect(hello)}") {type, {user, tenant_or_alias, db_name}} = HandlerHelpers.parse_user_info(hello.payload) - # Validate user and db_name according to PostgreSQL rules. - # The rules are: 1-63 characters, alphanumeric, underscore and $ - # TODO: spaces are allowed in db_name, but we don't support it yet - rule = ~r/^[a-z_][a-z0-9_$]*$/ - - if user =~ rule and db_name =~ rule do + if Helpers.validate_name(user) and Helpers.validate_name(db_name) do log_level = maybe_change_log(hello) search_path = hello.payload["options"]["--search_path"] event = {:hello, {type, {user, tenant_or_alias, db_name, search_path}}} diff --git a/lib/supavisor/helpers.ex b/lib/supavisor/helpers.ex index dc8bf8e4..7cc4364f 100644 --- a/lib/supavisor/helpers.ex +++ b/lib/supavisor/helpers.ex @@ -368,4 +368,12 @@ defmodule Supavisor.Helpers do @spec controlling_process(Supavisor.sock(), pid) :: :ok | {:error, any()} def controlling_process({mod, socket}, pid), do: mod.controlling_process(socket, pid) + + @spec validate_name(String.t()) :: boolean() + def validate_name(name) do + # 1-63 characters, starting with a lowercase letter or underscore, and containing only alphanumeric characters, underscores, and dollar signs. Names with spaces or uppercase letters must be enclosed in double quotes. + String.length(name) <= 63 and + name =~ ~r/^(?:[a-z_][a-z0-9_$ ]*|"[a-zA-Z0-9_$ ]+")$/ and + name != ~s/""/ + end end diff --git a/test/supavisor/helpers_test.exs b/test/supavisor/helpers_test.exs index 3966d1ad..4d2fb5a3 100644 --- a/test/supavisor/helpers_test.exs +++ b/test/supavisor/helpers_test.exs @@ -35,4 +35,52 @@ defmodule Supavisor.HelpersTest do {:error, "Unsupported or invalid secret format"} end end + + describe "validate_name/1" do + test "allows valid unquoted names" do + assert Helpers.validate_name("valid_name") + # Minimum length + assert Helpers.validate_name("a") + assert Helpers.validate_name("valid_name_123") + assert Helpers.validate_name("name$123") + end + + test "rejects invalid unquoted names" do + # Empty name + refute Helpers.validate_name("") + # Starts with a number + refute Helpers.validate_name("0invalid") + # Contains uppercase letters + refute Helpers.validate_name("InvalidName") + # Contains hyphen + refute Helpers.validate_name("invalid-name") + # Contains period + refute Helpers.validate_name("invalid.name") + # Over 63 chars + refute Helpers.validate_name( + "this_name_is_way_toooooo_long_and_exceeds_sixty_three_characters" + ) + end + + test "allows valid quoted names" do + # Contains space + assert Helpers.validate_name("\"Valid Name\"") + # Contains uppercase letters + assert Helpers.validate_name("\"ValidName123\"") + # Same as unquoted but quoted + assert Helpers.validate_name("\"valid_name\"") + # Contains dollar sign + assert Helpers.validate_name("\"Name with $\"") + assert Helpers.validate_name("\"name with multiple spaces\"") + end + + test "rejects invalid quoted names" do + # Contains hyphen + refute Helpers.validate_name("\"invalid-name\"") + # Contains period + refute Helpers.validate_name("\"invalid.name\"") + # Empty name + refute Helpers.validate_name("\"\"") + end + end end From a6cb71f486729cc529af04a4f9205a1dd2aaf676 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 30 Oct 2024 21:32:17 +0100 Subject: [PATCH 74/97] chore: update poolboy to v0.0.3 and enhance error logging and (#466) --- lib/supavisor/client_handler.ex | 2 +- lib/supavisor/secret_checker.ex | 2 -- mix.exs | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 31a70ea2..cc4d3916 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -194,7 +194,7 @@ defmodule Supavisor.ClientHandler do {:next_event, :internal, event}} else reason = "Invalid format for user or db_name" - Logger.error("ClientHandler: #{inspect(reason)}") + Logger.error("ClientHandler: #{inspect(reason)} #{inspect({user, db_name})}") Telem.client_join(:fail, tenant_or_alias) HandlerHelpers.send_error( diff --git a/lib/supavisor/secret_checker.ex b/lib/supavisor/secret_checker.ex index 3e02043b..78e9e607 100644 --- a/lib/supavisor/secret_checker.ex +++ b/lib/supavisor/secret_checker.ex @@ -68,9 +68,7 @@ defmodule Supavisor.SecretChecker do end def handle_info(:check, state) do - Logger.debug("Checking secrets") check_secrets(state) - Logger.debug("Secrets checked") {:noreply, %{state | check_ref: check()}} end diff --git a/mix.exs b/mix.exs index 4f3bc45f..bf04f113 100644 --- a/mix.exs +++ b/mix.exs @@ -68,7 +68,7 @@ defmodule Supavisor.MixProject do # pooller # {:poolboy, "~> 1.5.2"}, - {:poolboy, git: "https://github.com/abc3/poolboy.git", tag: "v0.0.2"}, + {:poolboy, git: "https://github.com/abc3/poolboy.git", tag: "v0.0.3"}, {:syn, "~> 3.3"}, {:pgo, "~> 0.13"}, {:rustler, "~> 0.34.0"}, From 3893fca5611fd008d56de4896429d8e8d0a80172 Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 4 Nov 2024 14:06:51 +0100 Subject: [PATCH 75/97] fix: update db_pid structure in terminate function (#467) --- lib/supavisor/client_handler.ex | 2 +- lib/supavisor/db_handler.ex | 5 ++++- mix.exs | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index cc4d3916..ab49ee38 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -707,7 +707,7 @@ defmodule Supavisor.ClientHandler do :ok end - def terminate(reason, _state, %{db_pid: {_, pid}}) do + def terminate(reason, _state, %{db_pid: {_, pid, _}}) do db_info = with {:ok, {state, mode} = resp} <- DbHandler.get_state_and_mode(pid) do if state == :busy or mode == :session, do: DbHandler.stop(pid) diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 6e0d5c51..e747d4d5 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -42,7 +42,10 @@ defmodule Supavisor.DbHandler do end @spec stop(pid()) :: :ok - def stop(pid), do: :gen_statem.stop(pid, :client_termination, 5_000) + def stop(pid) do + Logger.debug("DbHandler: Stop pid #{inspect(pid)}") + :gen_statem.stop(pid, :client_termination, 5_000) + end @impl true def init(args) do diff --git a/mix.exs b/mix.exs index bf04f113..4f3bc45f 100644 --- a/mix.exs +++ b/mix.exs @@ -68,7 +68,7 @@ defmodule Supavisor.MixProject do # pooller # {:poolboy, "~> 1.5.2"}, - {:poolboy, git: "https://github.com/abc3/poolboy.git", tag: "v0.0.3"}, + {:poolboy, git: "https://github.com/abc3/poolboy.git", tag: "v0.0.2"}, {:syn, "~> 3.3"}, {:pgo, "~> 0.13"}, {:rustler, "~> 0.34.0"}, From 07b40080e80a2e29b4852eeaada8cf7ec9912741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 5 Nov 2024 16:57:49 +0100 Subject: [PATCH 76/97] test: add assertion testing if value passes eventually (#470) --- test/supavisor/prom_ex_test.exs | 6 ++--- test/support/asserts.ex | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 test/support/asserts.ex diff --git a/test/supavisor/prom_ex_test.exs b/test/supavisor/prom_ex_test.exs index df684260..b489f44f 100644 --- a/test/supavisor/prom_ex_test.exs +++ b/test/supavisor/prom_ex_test.exs @@ -2,6 +2,8 @@ defmodule Supavisor.PromExTest do use ExUnit.Case, async: true use Supavisor.DataCase + import Supavisor.Asserts + alias Supavisor.Monitoring.PromEx @tenant "prom_tenant" @@ -31,9 +33,7 @@ defmodule Supavisor.PromExTest do :ok = GenServer.stop(proxy) :ok = Supavisor.stop({{:single, @tenant}, user, :transaction, db_name, nil}) - Process.sleep(3000) - - refute PromEx.get_metrics() =~ "tenant=\"#{@tenant}\"" + refute_eventually(10, fn -> PromEx.get_metrics() =~ "tenant=\"#{@tenant}\"" end) end test "clean_string/1 removes extra spaces from metric string" do diff --git a/test/support/asserts.ex b/test/support/asserts.ex new file mode 100644 index 00000000..d0e720b9 --- /dev/null +++ b/test/support/asserts.ex @@ -0,0 +1,41 @@ +defmodule Supavisor.Asserts do + @doc """ + Asserts that `function` will eventually success. Fails otherwise. + + It performs `repeats` checks with `delay` milliseconds between each check. + """ + def assert_eventually(repeats \\ 5, delay \\ 1000, function) + + def assert_eventually(0, _, _) do + raise ExUnit.AssertionError, message: "Expected function to return truthy value" + end + + def assert_eventually(n, delay, func) do + if func.() do + :ok + else + Process.sleep(delay) + assert_eventually(n - 1, delay, func) + end + end + + @doc """ + Asserts that `function` will eventually fail. Fails otherwise. + + It performs `repeats` checks with `delay` milliseconds between each check. + """ + def refute_eventually(repeats \\ 5, delay \\ 1000, function) + + def refute_eventually(0, _, _) do + raise ExUnit.AssertionError, message: "Expected function to return falsey value" + end + + def refute_eventually(n, delay, func) do + if func.() do + Process.sleep(delay) + refute_eventually(n - 1, delay, func) + else + :ok + end + end +end From 0f770a9c79b3d5bfe680aebc735bdc06edb547ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 5 Nov 2024 16:59:23 +0100 Subject: [PATCH 77/97] chore: check if lockfile is up to date (#469) This will raise if there are some pending changes to the `mix.lock`. This is to ensure, that `mix.lock` was uploaded after dependency changes. --- .github/workflows/elixir.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 95ca8314..ae1c23cc 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -36,7 +36,7 @@ jobs: restore-keys: | ${{ runner.os }}-mix- - name: Install dependencies - run: mix deps.get + run: mix deps.get --check-locked format: name: Formatting checks From 3222cd95bb1c7f6610850555bcfeeb869ca240a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Thu, 7 Nov 2024 19:12:05 +0100 Subject: [PATCH 78/97] chore: set behaviour that is implemented by SynHandler module (#472) --- lib/supavisor/syn_handler.ex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/supavisor/syn_handler.ex b/lib/supavisor/syn_handler.ex index 49450b4e..d4d2986b 100644 --- a/lib/supavisor/syn_handler.ex +++ b/lib/supavisor/syn_handler.ex @@ -2,9 +2,14 @@ defmodule Supavisor.SynHandler do @moduledoc """ Custom defined Syn's callbacks """ + + @behaviour :syn_event_handler + require Logger + alias Supavisor.Monitoring.PromEx + @impl true def on_process_unregistered( :tenants, {{_type, _tenant}, _user, _mode, _db_name, _search_path} = id, @@ -35,6 +40,7 @@ defmodule Supavisor.SynHandler do PromEx.remove_metrics(id) end + @impl true def resolve_registry_conflict( :tenants, id, From a79f2ebbf3676d137a960302e57bf3de274879c0 Mon Sep 17 00:00:00 2001 From: Chase Granberry Date: Fri, 8 Nov 2024 06:12:58 -0700 Subject: [PATCH 79/97] fix: max clients in session mode error (#473) * fix: update max clients error in session mode --- docs/monitoring/logs.md | 9 +++ lib/supavisor/client_handler.ex | 2 +- lib/supavisor/manager.ex | 2 +- mkdocs.yaml | 89 +++++++++++++++-------------- priv/repo/seeds_after_migration.exs | 2 +- test/integration/proxy_test.exs | 10 +++- 6 files changed, 66 insertions(+), 48 deletions(-) create mode 100644 docs/monitoring/logs.md diff --git a/docs/monitoring/logs.md b/docs/monitoring/logs.md new file mode 100644 index 00000000..1858a04f --- /dev/null +++ b/docs/monitoring/logs.md @@ -0,0 +1,9 @@ +Supavisor will emit various logs during operation. + +Use these error codes to debug a running Supavisor cluster. + +## Error Codes + +| Code | Description | +| ----------------------- | -------------------------------------------------------------------- | +| MaxClientsInSessionMode | When in Session mode client connections are limited by the pool_size | diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index ab49ee38..03444a1d 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -696,7 +696,7 @@ defmodule Supavisor.ClientHandler do msg = case data.mode do :session -> - "Max client connections reached" + "MaxClientsInSessionMode: max clients reached - in Session mode max clients are limited to pool_size" :transaction -> "Unable to check out process from the pool due to timeout" diff --git a/lib/supavisor/manager.ex b/lib/supavisor/manager.ex index ee86f7f8..079e4898 100644 --- a/lib/supavisor/manager.ex +++ b/lib/supavisor/manager.ex @@ -65,7 +65,7 @@ defmodule Supavisor.Manager do # don't limit if max_clients is null {reply, new_state} = - if :ets.info(state.tid, :size) < state.max_clients do + if :ets.info(state.tid, :size) < state.max_clients or Supavisor.mode(state.id) == :session do :ets.insert(state.tid, {Process.monitor(pid), pid, now()}) case state.parameter_status do diff --git a/mkdocs.yaml b/mkdocs.yaml index 72369950..436b4be5 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -6,51 +6,52 @@ repo_name: supabase/supavisor repo_url: https://github.com/supabase/supavisor nav: - - Welcome: 'index.md' - - FAQ: 'faq.md' - - Development: - - Installation: 'development/installation.md' - - Setup: 'development/setup.md' - - Docs: 'development/docs.md' - - Deployment: - - Deploy with Fly.io: 'deployment/fly.md' - - Connecting: - - Overview: 'connecting/overview.md' - - Authentication: 'connecting/authentication.md' - - Configuration: - - Tenants: 'configuration/tenants.md' - - Users: 'configuration/users.md' - - Pool Modes: 'configuration/pool_modes.md' - - Migrating: - - from PgBouncer: 'migrating/pgbouncer.md' - - Monitoring: - - Metrics: 'monitoring/metrics.md' - - ORMs: - - Prisma: 'orms/prisma.md' + - Welcome: "index.md" + - FAQ: "faq.md" + - Development: + - Installation: "development/installation.md" + - Setup: "development/setup.md" + - Docs: "development/docs.md" + - Deployment: + - Deploy with Fly.io: "deployment/fly.md" + - Connecting: + - Overview: "connecting/overview.md" + - Authentication: "connecting/authentication.md" + - Configuration: + - Tenants: "configuration/tenants.md" + - Users: "configuration/users.md" + - Pool Modes: "configuration/pool_modes.md" + - Migrating: + - from PgBouncer: "migrating/pgbouncer.md" + - Monitoring: + - Metrics: "monitoring/metrics.md" + - Logs: "monitoring/logs.md" + - ORMs: + - Prisma: "orms/prisma.md" theme: - name: 'material' - favicon: 'images/favicon.ico' - logo: 'images/favicon.ico' - homepage: https://supabase.github.io/supavisor - features: - - navigation.expand - palette: - primary: black - accent: light green + name: "material" + favicon: "images/favicon.ico" + logo: "images/favicon.ico" + homepage: https://supabase.github.io/supavisor + features: + - navigation.expand + palette: + primary: black + accent: light green markdown_extensions: - - pymdownx.highlight: - linenums: true - guess_lang: false - use_pygments: true - pygments_style: default - - pymdownx.superfences - - pymdownx.tabbed: - alternate_style: true - - pymdownx.snippets - - pymdownx.tasklist - - admonition - - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg \ No newline at end of file + - pymdownx.highlight: + linenums: true + guess_lang: false + use_pygments: true + pygments_style: default + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.snippets + - pymdownx.tasklist + - admonition + - pymdownx.emoji: + emoji_index: !!python/name:materialx.emoji.twemoji + emoji_generator: !!python/name:materialx.emoji.to_svg diff --git a/priv/repo/seeds_after_migration.exs b/priv/repo/seeds_after_migration.exs index e244c472..424382b0 100644 --- a/priv/repo/seeds_after_migration.exs +++ b/priv/repo/seeds_after_migration.exs @@ -78,7 +78,7 @@ end "db_user_alias" => "max_clients", "db_user" => db_conf[:username], "db_password" => db_conf[:password], - "pool_size" => 2, + "pool_size" => 1, "max_clients" => -1, "mode_type" => "transaction", "pool_checkout_timeout" => 500 diff --git a/test/integration/proxy_test.exs b/test/integration/proxy_test.exs index cd9b9619..023e35b9 100644 --- a/test/integration/proxy_test.exs +++ b/test/integration/proxy_test.exs @@ -166,16 +166,24 @@ defmodule Supavisor.Integration.ProxyTest do port: port ) + {:ok, pid1} = single_connection(connection_opts) + {:ok, pid2} = single_connection(connection_opts) + + :timer.sleep(1000) + assert {:error, %Postgrex.Error{ postgres: %{ code: :internal_error, - message: "Max client connections reached", + message: + "MaxClientsInSessionMode: max clients reached - in Session mode max clients are limited to pool_size", unknown: "FATAL", severity: "FATAL", pg_code: "XX000" } }} = single_connection(connection_opts) + + for pid <- [pid1, pid2], do: :gen_statem.stop(pid) end test "http to proxy server returns 200 OK" do From adfe5bf69ca0f4f668d758854e2ae83ec581be23 Mon Sep 17 00:00:00 2001 From: Stas Date: Tue, 12 Nov 2024 13:24:15 +0100 Subject: [PATCH 80/97] fix: better handling of resetting active_counter (#471) --- lib/supavisor/client_handler.ex | 25 +++++++++++++++++-------- lib/supavisor/db_handler.ex | 12 ++++++------ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 03444a1d..764fcfe7 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -519,21 +519,21 @@ defmodule Supavisor.ClientHandler do do: :ok = sock_send_maybe_active_once(msg, data), else: :ok = HandlerHelpers.sock_send(data.sock, Server.ready_for_query()) - {:keep_state, %{data | active_count: 0}, handle_actions(data)} + {:keep_state, %{data | active_count: reset_active_count(data)}, handle_actions(data)} end def handle_event(:info, {proto, _, <> = msg}, _, data) when proto in @proto do Logger.debug("ClientHandler: Receive sync while not idle") :ok = sock_send_maybe_active_once(msg, data) - {:keep_state, %{data | active_count: 0}, handle_actions(data)} + {:keep_state, %{data | active_count: reset_active_count(data)}, handle_actions(data)} end def handle_event(:info, {proto, _, <> = msg}, _, data) when proto in @proto do Logger.debug("ClientHandler: Receive flush while not idle") :ok = sock_send_maybe_active_once(msg, data) - {:keep_state, %{data | active_count: 0}, handle_actions(data)} + {:keep_state, %{data | active_count: reset_active_count(data)}, handle_actions(data)} end # incoming query with a single pool @@ -645,17 +645,16 @@ defmodule Supavisor.ClientHandler do :ready_for_query -> Logger.debug("ClientHandler: Client is ready") - HandlerHelpers.sock_send(data.sock, bin) + :ok = sock_send_maybe_active_once(bin, data) + db_pid = handle_db_pid(data.mode, data.pool, data.db_pid) {_, stats} = Telem.network_usage(:client, data.sock, data.id, data.stats) Telem.client_query_time(data.query_start, data.id) - if data.active_count > @switch_active_count, - do: HandlerHelpers.activate(data.sock) - - {:next_state, :idle, %{data | db_pid: db_pid, stats: stats, active_count: 0}, + {:next_state, :idle, + %{data | db_pid: db_pid, stats: stats, active_count: reset_active_count(data)}, handle_actions(data)} :read_sql_error -> @@ -1123,4 +1122,14 @@ defmodule Supavisor.ClientHandler do {:stop, {:shutdown, :subscribe_retries}} end end + + @spec reset_active_count(map()) :: 0 + def reset_active_count(data) do + if data.active_count >= @switch_active_count do + Logger.debug("ClientHandler: Activate socket #{inspect(data.active_count)}") + HandlerHelpers.activate(data.sock) + end + + 0 + end end diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index e747d4d5..1cc2a8bd 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -294,10 +294,10 @@ defmodule Supavisor.DbHandler do when is_pid(caller) and proto in @proto do Logger.debug("DbHandler: Got write replica message #{inspect(bin)}") - if data.active_count > @switch_active_count, - do: HandlerHelpers.active_once(data.sock) - if String.ends_with?(bin, Server.ready_for_query()) do + if data.active_count >= @switch_active_count, + do: HandlerHelpers.activate(data.sock) + {_, stats} = Telem.network_usage(:db, data.sock, data.id, data.stats) # in transaction mode, we need to notify the client when the transaction is finished, @@ -311,11 +311,11 @@ defmodule Supavisor.DbHandler do %{data | stats: stats, active_count: 0} end - if data.active_count > @switch_active_count, - do: HandlerHelpers.activate(data.sock) - {:next_state, :idle, data, {:next_event, :internal, :check_anon_buffer}} else + if data.active_count > @switch_active_count, + do: HandlerHelpers.active_once(data.sock) + HandlerHelpers.sock_send(data.client_sock, bin) {:keep_state, %{data | active_count: data.active_count + 1}} end From ed031710a45bb12995a85f09ea09666555a3b994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 12 Nov 2024 13:25:11 +0100 Subject: [PATCH 81/97] test: mark PromEx test as flaky (#474) That tests are constantly causing us problems in CI, but as it is test for internal functionality that do not affect users directly. So to streamline testing disable running these tests in default run. To run them one can use mix test --include flaky --- test/supavisor/prom_ex_test.exs | 6 ++++++ test/test_helper.exs | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test/supavisor/prom_ex_test.exs b/test/supavisor/prom_ex_test.exs index b489f44f..27edfc0c 100644 --- a/test/supavisor/prom_ex_test.exs +++ b/test/supavisor/prom_ex_test.exs @@ -8,6 +8,12 @@ defmodule Supavisor.PromExTest do @tenant "prom_tenant" + # These tests are known to be flaky, and while these do not affect users + # directly we can run them independently when needed. In future we probably + # should make them pass "regularly", but for now that is easier to filter them + # out. + @moduletag flaky: true + setup do db_conf = Application.get_env(:supavisor, Repo) diff --git a/test/test_helper.exs b/test/test_helper.exs index 20539b66..d7449a11 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -2,5 +2,9 @@ Cachex.start_link(name: Supavisor.Cache) -ExUnit.start(capture_log: true) +ExUnit.start( + capture_log: true, + exclude: [flaky: true] +) + Ecto.Adapters.SQL.Sandbox.mode(Supavisor.Repo, :auto) From 91fcf4e59b591b87bbb81a29792cdfdce630045b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 12 Nov 2024 19:42:02 +0100 Subject: [PATCH 82/97] fix: mark `:db_termination` as shutdown (#475) --- lib/supavisor/db_handler.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 1cc2a8bd..0c418beb 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -343,7 +343,7 @@ defmodule Supavisor.DbHandler do end def handle_event(_, {closed, _}, :busy, data) when closed in @sock_closed do - {:stop, :db_termination, data} + {:stop, {:shutdown, :db_termination}, data} end def handle_event(_, {closed, _}, state, data) when closed in @sock_closed do @@ -351,7 +351,7 @@ defmodule Supavisor.DbHandler do if Application.get_env(:supavisor, :reconnect_on_db_close), do: {:next_state, :connect, data, {:state_timeout, reconnect_timeout(data), :connect}}, - else: {:stop, :db_termination, data} + else: {:stop, {:shutdown, :db_termination}, data} end # linked client_handler went down From 485688767e65112af424322838ead2e9b5c6db11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 12 Nov 2024 19:42:20 +0100 Subject: [PATCH 83/97] fix: `DbHandler.stop/1` should exit with normal reason (#476) --- lib/supavisor/db_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 0c418beb..492aadf2 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -44,7 +44,7 @@ defmodule Supavisor.DbHandler do @spec stop(pid()) :: :ok def stop(pid) do Logger.debug("DbHandler: Stop pid #{inspect(pid)}") - :gen_statem.stop(pid, :client_termination, 5_000) + :gen_statem.stop(pid, {:shutdown, :client_termination}, 5_000) end @impl true From e02207f60374fb9e8bb7849ef4a27c5b13a8874e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 12 Nov 2024 19:42:39 +0100 Subject: [PATCH 84/97] chore: update Hex dependencies (#477) --- mix.lock | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/mix.lock b/mix.lock index bd1d50d1..63a3aa0c 100644 --- a/mix.lock +++ b/mix.lock @@ -5,29 +5,29 @@ "bertex": {:hex, :bertex, "1.3.0", "0ad0df9159b5110d9d2b6654f72fbf42a54884ef43b6b651e6224c0af30ba3cb", [:mix], [], "hexpm", "0a5d5e478bb5764b7b7bae37cae1ca491200e58b089df121a2fe1c223d8ee57a"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"}, - "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, + "castore": {:hex, :castore, "1.0.9", "5cc77474afadf02c7c017823f460a17daa7908e991b0cc917febc90e466a375c", [:mix], [], "hexpm", "5ea956504f1ba6f2b4eb707061d8e17870de2bee95fb59d512872c2ef06925e7"}, "cloak": {:hex, :cloak, "1.1.4", "aba387b22ea4d80d92d38ab1890cc528b06e0e7ef2a4581d71c3fdad59e997e7", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "92b20527b9aba3d939fab0dd32ce592ff86361547cfdc87d74edce6f980eb3d7"}, "cloak_ecto": {:hex, :cloak_ecto, "1.3.0", "0de127c857d7452ba3c3367f53fb814b0410ff9c680a8d20fbe8b9a3c57a1118", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "314beb0c123b8a800418ca1d51065b27ba3b15f085977e65c0f7b2adab2de1cc"}, "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, - "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, + "credo": {:hex, :credo, "1.7.10", "6e64fe59be8da5e30a1b96273b247b5cf1cc9e336b5fd66302a64b25749ad44d", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "71fbc9a6b8be21d993deca85bf151df023a3097b01e09a2809d460348561d8cd"}, "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, + "dialyxir": {:hex, :dialyxir, "1.4.4", "fb3ce8741edeaea59c9ae84d5cec75da00fa89fe401c72d6e047d11a61f65f70", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "cd6111e8017ccd563e65621a4d9a4a1c5cd333df30cebc7face8029cacb4eff6"}, "distillery": {:hex, :distillery, "2.1.1", "f9332afc2eec8a1a2b86f22429e068ef35f84a93ea1718265e740d90dd367814", [:mix], [{:artificery, "~> 0.2", [hex: :artificery, repo: "hexpm", optional: false]}], "hexpm", "bbc7008b0161a6f130d8d903b5b3232351fccc9c31a991f8fcbf2a12ace22995"}, - "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, - "ecto_sql": {:hex, :ecto_sql, "3.11.3", "4eb7348ff8101fbc4e6bbc5a4404a24fecbe73a3372d16569526b0cf34ebc195", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e5f36e3d736b99c7fee3e631333b8394ade4bafe9d96d35669fca2d81c2be928"}, + "ecto": {:hex, :ecto, "3.12.4", "267c94d9f2969e6acc4dd5e3e3af5b05cdae89a4d549925f3008b2b7eb0b93c3", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ef04e4101688a67d061e1b10d7bc1fbf00d1d13c17eef08b71d070ff9188f747"}, + "ecto_sql": {:hex, :ecto_sql, "3.12.1", "c0d0d60e85d9ff4631f12bafa454bc392ce8b9ec83531a412c12a0d415a3a4d0", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aff5b958a899762c5f09028c847569f7dfb9cc9d63bdb8133bff8a5546de6bf5"}, "eflambe": {:hex, :eflambe, "0.3.1", "ef0a35084fad1f50744496730a9662782c0a9ebf449d3e03143e23295c5926ea", [:rebar3], [{:meck, "0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "58d5997be606d4e269e9e9705338e055281fdf3e4935cc902c8908e9e4516c5f"}, "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, - "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, - "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, + "file_system": {:hex, :file_system, "1.0.1", "79e8ceaddb0416f8b8cd02a0127bdbababe7bf4a23d2a395b983c1f8b3f73edd", [:mix], [], "hexpm", "4414d1f38863ddf9120720cd976fce5bdde8e91d8283353f0e31850fa89feb9e"}, + "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"}, "inet_cidr": {:hex, :inet_cidr, "1.0.8", "d26bb7bdbdf21ae401ead2092bf2bb4bf57fe44a62f5eaa5025280720ace8a40", [:mix], [], "hexpm", "d5b26da66603bb56c933c65214c72152f0de9a6ea53618b56d63302a68f6a90e"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, - "joken": {:hex, :joken, "2.6.1", "2ca3d8d7f83bf7196296a3d9b2ecda421a404634bfc618159981a960020480a1", [:mix], [{:jose, "~> 1.11.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "ab26122c400b3d254ce7d86ed066d6afad27e70416df947cdcb01e13a7382e68"}, + "joken": {:hex, :joken, "2.6.2", "5daaf82259ca603af4f0b065475099ada1b2b849ff140ccd37f4b6828ca6892a", [:mix], [{:jose, "~> 1.11.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5134b5b0a6e37494e46dbf9e4dad53808e5e787904b7c73972651b51cce3d72b"}, "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, "jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"}, "libcluster": {:hex, :libcluster, "3.3.3", "a4f17721a19004cfc4467268e17cff8b1f951befe428975dd4f6f7b84d927fe0", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "7c0a2275a0bb83c07acd17dab3c3bfb4897b145106750eeccc62d302e3bdfee5"}, @@ -39,16 +39,16 @@ "mint": {:hex, :mint, "1.6.2", "af6d97a4051eee4f05b5500671d47c3a67dac7386045d87a904126fd4bbcea2e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5ee441dffc1892f1ae59127f74afe8fd82fda6587794278d924e4d90ea3d63f9"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, - "observer_cli": {:hex, :observer_cli, "1.7.4", "3c1bfb6d91bf68f6a3d15f46ae20da0f7740d363ee5bc041191ce8722a6c4fae", [:mix, :rebar3], [{:recon, "~> 2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "50de6d95d814f447458bd5d72666a74624eddb0ef98bdcee61a0153aae0865ff"}, + "observer_cli": {:hex, :observer_cli, "1.8.0", "1359409c4b25b11360db56bc3103cfb51f3a4b3aea76ec58c7b8595feb5d6019", [:mix, :rebar3], [{:recon, "~> 2.5.6", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "9842759b11360819dd0e6e60173c39c1e6aaef4b20fa6fe9b4700e3e02911b83"}, "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"}, - "open_api_spex": {:hex, :open_api_spex, "3.20.0", "d4fcf1ee297aa94a673cddb92734eb0bc7cac698be93949a223a50f724e3af89", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "2e9beea71142ff09f8f935579b39406e2c6b5a3978e7235978d7faf2f90cd081"}, - "opentelemetry_api": {:hex, :opentelemetry_api, "1.3.0", "03e2177f28dd8d11aaa88e8522c81c2f6a788170fe52f7a65262340961e663f9", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "b9e5ff775fd064fa098dba3c398490b77649a352b40b0b730a6b7dc0bdd68858"}, + "open_api_spex": {:hex, :open_api_spex, "3.21.2", "6a704f3777761feeb5657340250d6d7332c545755116ca98f33d4b875777e1e5", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "f42ae6ed668b895ebba3e02773cfb4b41050df26f803f2ef634c72a7687dc387"}, + "opentelemetry_api": {:hex, :opentelemetry_api, "1.4.0", "63ca1742f92f00059298f478048dfb826f4b20d49534493d6919a0db39b6db04", [:mix, :rebar3], [], "hexpm", "3dfbbfaa2c2ed3121c5c483162836c4f9027def469c41578af5ef32589fcfc58"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, - "peep": {:hex, :peep, "3.1.0", "1680337d682dfde308b643814834379a5210eb11db5aaca8ea823c218d4d7e16", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "3de3c13f3efcff6130e53c8956d41c20a436650cfddee6e8aa6ebdef751d542f"}, + "peep": {:hex, :peep, "3.3.0", "ece8c38f0e3cfeecf8739d377c228c7e2b34d947f6b4817a183be37c88b94ebe", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:plug, "~> 1.16", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "19c84bf78c4eee97cb0df33d4ea628e93b6ab148ee19c289c499d1d62b3e78cb"}, "pg_types": {:hex, :pg_types, "0.4.0", "3ce365c92903c5bb59c0d56382d842c8c610c1b6f165e20c4b652c96fa7e9c14", [:rebar3], [], "hexpm", "b02efa785caececf9702c681c80a9ca12a39f9161a846ce17b01fb20aeeed7eb"}, "pgo": {:hex, :pgo, "0.14.0", "f53711d103d7565db6fc6061fcf4ff1007ab39892439be1bb02d9f686d7e6663", [:rebar3], [{:backoff, "~> 1.1.6", [hex: :backoff, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:pg_types, "~> 0.4.0", [hex: :pg_types, repo: "hexpm", optional: false]}], "hexpm", "71016c22599936e042dc0012ee4589d24c71427d266292f775ebf201d97df9c9"}, "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"}, - "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.2", "3b83b24ab5a2eb071a20372f740d7118767c272db386831b2e77638c4dcc606d", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "3f94d025f59de86be00f5f8c5dd7b5965a3298458d21ab1c328488be3b5fcd59"}, + "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.3", "f686701b0499a07f2e3b122d84d52ff8a31f5def386e03706c916f6feddf69ef", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "909502956916a657a197f94cc1206d9a65247538de8a5e186f7537c895d95764"}, "phoenix_html": {:hex, :phoenix_html, "4.1.1", "4c064fd3873d12ebb1388425a8f2a19348cef56e7289e1998e2d2fa758aa982e", [:mix], [], "hexpm", "f2f2df5a72bc9a2f510b21497fd7d2b86d932ec0598f0210fed4114adc546c6f"}, "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.4", "4508e481f791ce62ec6a096e13b061387158cbeefacca68c6c1928e1305e23ed", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "2984aae96994fbc5c61795a73b8fb58153b41ff934019cfb522343d2d3817d59"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.5.3", "f2161c207fda0e4fb55165f650f7f8db23f02b29e3bff00ff7ef161d6ac1f09d", [:mix], [{:file_system, "~> 0.3 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "b4ec9cd73cb01ff1bd1cac92e045d13e7030330b74164297d1aee3907b54803c"}, @@ -57,27 +57,27 @@ "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, "phoenix_view": {:hex, :phoenix_view, "2.0.4", "b45c9d9cf15b3a1af5fb555c674b525391b6a1fe975f040fb4d913397b31abf4", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "4e992022ce14f31fe57335db27a28154afcc94e9983266835bb3040243eb620b"}, "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.7.2", "fdadb973799ae691bf9ecad99125b16625b1c6039999da5fe544d99218e662e4", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "245d8a11ee2306094840c000e8816f0cbed69a23fc0ac2bcf8d7835ae019bb2f"}, "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "poolboy": {:git, "https://github.com/abc3/poolboy.git", "999ec7f5c7282d515020bb058b4832029d6d07bc", [tag: "v0.0.2"]}, - "postgrex": {:hex, :postgrex, "0.19.0", "f7d50e50cb42e0a185f5b9a6095125a9ab7e4abccfbe2ab820ab9aa92b71dbab", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "dba2d2a0a8637defbf2307e8629cb2526388ba7348f67d04ec77a5d6a72ecfae"}, - "prom_ex": {:hex, :prom_ex, "1.10.0", "2fe14cd0b2f7f8688280b02861c58da9624571f82a878b203825fb2bec206195", [:mix], [{:absinthe, ">= 1.7.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.1.0", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.11.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.18", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.10.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.4", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:peep, "~> 3.0", [hex: :peep, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.7.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.20.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.16.0", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 2.6.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.2", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.1", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "55d6ca87519b14202570dda5f29bf3f52e14b940eaf2ddcd89ad41347fa5781a"}, + "postgrex": {:hex, :postgrex, "0.19.3", "a0bda6e3bc75ec07fca5b0a89bffd242ca209a4822a9533e7d3e84ee80707e19", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d31c28053655b78f47f948c85bb1cf86a9c1f8ead346ba1aa0d0df017fa05b61"}, + "prom_ex": {:hex, :prom_ex, "1.11.0", "1f6d67f2dead92224cb4f59beb3e4d319257c5728d9638b4a5e8ceb51a4f9c7e", [:mix], [{:absinthe, ">= 1.7.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.1.0", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.11.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.18", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.10.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.4", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:peep, "~> 3.0", [hex: :peep, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.7.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.20.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.16.0", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 2.6.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.2", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.1", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "76b074bc3730f0802978a7eb5c7091a65473eaaf07e99ec9e933138dcc327805"}, "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, - "recon": {:hex, :recon, "2.5.5", "c108a4c406fa301a529151a3bb53158cadc4064ec0c5f99b03ddb8c0e4281bdf", [:mix, :rebar3], [], "hexpm", "632a6f447df7ccc1a4a10bdcfce71514412b16660fe59deca0fcf0aa3c054404"}, - "req": {:hex, :req, "0.5.4", "e375e4812adf83ffcf787871d7a124d873e983e3b77466e6608b973582f7f837", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "a17998ffe2ef54f79bfdd782ef9f4cbf987d93851e89444cbc466a6a25eee494"}, + "recon": {:hex, :recon, "2.5.6", "9052588e83bfedfd9b72e1034532aee2a5369d9d9343b61aeb7fbce761010741", [:mix, :rebar3], [], "hexpm", "96c6799792d735cc0f0fd0f86267e9d351e63339cbe03df9d162010cefc26bb0"}, + "req": {:hex, :req, "0.5.7", "b722680e03d531a2947282adff474362a48a02aa54b131196fbf7acaff5e4cee", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "c6035374615120a8923e8089d0c21a3496cf9eda2d287b806081b8f323ceee29"}, "rustler": {:hex, :rustler, "0.34.0", "e9a73ee419fc296a10e49b415a2eb87a88c9217aa0275ec9f383d37eed290c1c", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "1d0c7449482b459513003230c0e2422b0252245776fe6fd6e41cb2b11bd8e628"}, "sleeplocks": {:hex, :sleeplocks, "1.1.3", "96a86460cc33b435c7310dbd27ec82ca2c1f24ae38e34f8edde97f756503441a", [:rebar3], [], "hexpm", "d3b3958552e6eb16f463921e70ae7c767519ef8f5be46d7696cc1ed649421321"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, "syn": {:hex, :syn, "3.3.0", "4684a909efdfea35ce75a9662fc523e4a8a4e8169a3df275e4de4fa63f99c486", [:rebar3], [], "hexpm", "e58ee447bc1094bdd21bf0acc102b1fbf99541a508cd48060bf783c245eaf7d6"}, - "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, + "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"}, "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"}, "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, - "tesla": {:hex, :tesla, "1.11.2", "24707ac48b52f72f88fc05d242b1c59a85d1ee6f16f19c312d7d3419665c9cd5", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "c549cd03aec6a7196a641689dd378b799e635eb393f689b4bd756f750c7a4014"}, + "tesla": {:hex, :tesla, "1.13.2", "85afa342eb2ac0fee830cf649dbd19179b6b359bec4710d02a3d5d587f016910", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:mox, "~> 1.0", [hex: :mox, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "960609848f1ef654c3cdfad68453cd84a5febecb6ed9fed9416e36cd9cd724f9"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, "unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, - "websock_adapter": {:hex, :websock_adapter, "0.5.6", "0437fe56e093fd4ac422de33bf8fc89f7bc1416a3f2d732d8b2c8fd54792fe60", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "e04378d26b0af627817ae84c92083b7e97aca3121196679b73c73b99d0d133ea"}, + "websock_adapter": {:hex, :websock_adapter, "0.5.7", "65fa74042530064ef0570b75b43f5c49bb8b235d6515671b3d250022cb8a1f9e", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d0f478ee64deddfec64b800673fd6e0c8888b079d9f3444dd96d2a98383bdbd1"}, } From 62b642aa9f7736bd18d2bf10974806550fbdca97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Wed, 13 Nov 2024 15:00:37 +0100 Subject: [PATCH 85/97] test: move HandlerHelpers tests to their module (#480) --- test/supavisor/client_handler_test.exs | 47 ---------------------- test/supavisor/handler_helpers_test.exs | 52 ++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 49 deletions(-) diff --git a/test/supavisor/client_handler_test.exs b/test/supavisor/client_handler_test.exs index 86314d18..37de47db 100644 --- a/test/supavisor/client_handler_test.exs +++ b/test/supavisor/client_handler_test.exs @@ -1,50 +1,3 @@ defmodule Supavisor.ClientHandlerTest do use ExUnit.Case, async: true - - alias Supavisor.HandlerHelpers, as: HH - - describe "parse_user_info/1" do - test "extracts the external_id from the username" do - payload = %{"user" => "test.user.external_id"} - {:single, {name, external_id, nil}} = HH.parse_user_info(payload) - assert name == "test.user" - assert external_id == "external_id" - end - - test "username consists only of username" do - username = "username" - payload = %{"user" => username} - {:single, {user, nil, nil}} = HH.parse_user_info(payload) - assert username == user - end - - test "consist cluster" do - username = "some.user.cluster.alias" - {t, {u, a, nil}} = HH.parse_user_info(%{"user" => username}) - assert {t, {u, a, nil}} == {:cluster, {"some.user", "alias", nil}} - end - - test "external_id in options" do - user = "test.user" - external_id = "external_id" - payload = %{"options" => %{"reference" => external_id}, "user" => user} - {:single, {user1, external_id1, nil}} = HH.parse_user_info(payload) - assert user1 == user - assert external_id1 == external_id - end - - test "unicode in username" do - payload = %{"user" => "тестовe.імʼя.external_id"} - {:single, {name, external_id, nil}} = HH.parse_user_info(payload) - assert name == "тестовe.імʼя" - assert external_id == "external_id" - end - - test "extracts db_name" do - payload = %{"user" => "user", "database" => "postgres_test"} - {:single, {name, nil, db_name}} = HH.parse_user_info(payload) - assert name == "user" - assert db_name == "postgres_test" - end - end end diff --git a/test/supavisor/handler_helpers_test.exs b/test/supavisor/handler_helpers_test.exs index 28a3512b..e1a1e090 100644 --- a/test/supavisor/handler_helpers_test.exs +++ b/test/supavisor/handler_helpers_test.exs @@ -1,4 +1,52 @@ defmodule Supavisor.HandlerHelpersTest do - use ExUnit.Case - doctest Supavisor.HandlerHelpers + use ExUnit.Case, async: true + + @subject Supavisor.HandlerHelpers + + doctest @subject + + describe "parse_user_info/1" do + test "extracts the external_id from the username" do + payload = %{"user" => "test.user.external_id"} + {:single, {name, external_id, nil}} = @subject.parse_user_info(payload) + assert name == "test.user" + assert external_id == "external_id" + end + + test "username consists only of username" do + username = "username" + payload = %{"user" => username} + {:single, {user, nil, nil}} = @subject.parse_user_info(payload) + assert username == user + end + + test "consist cluster" do + username = "some.user.cluster.alias" + {t, {u, a, nil}} = @subject.parse_user_info(%{"user" => username}) + assert {t, {u, a, nil}} == {:cluster, {"some.user", "alias", nil}} + end + + test "external_id in options" do + user = "test.user" + external_id = "external_id" + payload = %{"options" => %{"reference" => external_id}, "user" => user} + {:single, {user1, external_id1, nil}} = @subject.parse_user_info(payload) + assert user1 == user + assert external_id1 == external_id + end + + test "unicode in username" do + payload = %{"user" => "тестовe.імʼя.external_id"} + {:single, {name, external_id, nil}} = @subject.parse_user_info(payload) + assert name == "тестовe.імʼя" + assert external_id == "external_id" + end + + test "extracts db_name" do + payload = %{"user" => "user", "database" => "postgres_test"} + {:single, {name, nil, db_name}} = @subject.parse_user_info(payload) + assert name == "user" + assert db_name == "postgres_test" + end + end end From 9982a32ae7c6d9b2620d9c68cbff35c12002dc59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Wed, 13 Nov 2024 15:01:17 +0100 Subject: [PATCH 86/97] test: remove unneeded `match?/2` calls and instead use `=` operator (#478) --- test/integration/proxy_test.exs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/test/integration/proxy_test.exs b/test/integration/proxy_test.exs index 023e35b9..0f9ade02 100644 --- a/test/integration/proxy_test.exs +++ b/test/integration/proxy_test.exs @@ -1,5 +1,5 @@ defmodule Supavisor.Integration.ProxyTest do - use Supavisor.DataCase, async: true + use Supavisor.DataCase, async: false require Logger @@ -361,20 +361,17 @@ defmodule Supavisor.Integration.ProxyTest do id = {{:single, @tenant}, db_conf[:username], :session, db_conf[:database], nil} [{client_pid, _}] = Registry.lookup(Supavisor.Registry.TenantClients, id) - assert match?({_, %{active_count: 1}}, :sys.get_state(client_pid)) + assert {_, %{active_count: 1}} = :sys.get_state(client_pid) Enum.each(0..200, fn _ -> P.SimpleConnection.call(pid, {:query, "select 1;"}) end) - assert match?( - [ - %Postgrex.Result{ - command: :select - } - ], - P.SimpleConnection.call(pid, {:query, "select 1;"}) - ) + assert [ + %Postgrex.Result{ + command: :select + } + ] = P.SimpleConnection.call(pid, {:query, "select 1;"}) end defp single_connection(db_conf, c_port \\ nil) when is_list(db_conf) do From 7ec36273884ee8750edaf623406c28c66c5db13a Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 13 Nov 2024 16:27:53 +0100 Subject: [PATCH 87/97] fix: send a db message on ready_on_query (#482) --- lib/supavisor/client_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 764fcfe7..3e8226bf 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -645,7 +645,7 @@ defmodule Supavisor.ClientHandler do :ready_for_query -> Logger.debug("ClientHandler: Client is ready") - :ok = sock_send_maybe_active_once(bin, data) + :ok = HandlerHelpers.sock_send(data.sock, bin) db_pid = handle_db_pid(data.mode, data.pool, data.db_pid) From f054c35a2de3bb31bf4957feee246948114fc31b Mon Sep 17 00:00:00 2001 From: Stas Date: Thu, 14 Nov 2024 11:36:17 +0100 Subject: [PATCH 88/97] test: verify active_count increases after executing queries (#483) --- test/integration/proxy_test.exs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/proxy_test.exs b/test/integration/proxy_test.exs index 0f9ade02..68f15bb1 100644 --- a/test/integration/proxy_test.exs +++ b/test/integration/proxy_test.exs @@ -361,7 +361,9 @@ defmodule Supavisor.Integration.ProxyTest do id = {{:single, @tenant}, db_conf[:username], :session, db_conf[:database], nil} [{client_pid, _}] = Registry.lookup(Supavisor.Registry.TenantClients, id) - assert {_, %{active_count: 1}} = :sys.get_state(client_pid) + P.SimpleConnection.call(pid, {:query, "select 1;"}) + {_, %{active_count: active_count}} = :sys.get_state(client_pid) + assert active_count >= 1 Enum.each(0..200, fn _ -> P.SimpleConnection.call(pid, {:query, "select 1;"}) From bfba3fd437ecbc62c75825aab8a176101912a748 Mon Sep 17 00:00:00 2001 From: Stas Date: Thu, 14 Nov 2024 16:46:17 +0100 Subject: [PATCH 89/97] chore: improve active_count (#481) --- lib/supavisor/client_handler.ex | 7 ++----- lib/supavisor/db_handler.ex | 5 ++--- lib/supavisor/protocol/server.ex | 4 ++++ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 3e8226bf..6ab3ad4e 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -529,6 +529,7 @@ defmodule Supavisor.ClientHandler do {:keep_state, %{data | active_count: reset_active_count(data)}, handle_actions(data)} end + # handle Flush message def handle_event(:info, {proto, _, <> = msg}, _, data) when proto in @proto do Logger.debug("ClientHandler: Receive flush while not idle") @@ -1125,11 +1126,7 @@ defmodule Supavisor.ClientHandler do @spec reset_active_count(map()) :: 0 def reset_active_count(data) do - if data.active_count >= @switch_active_count do - Logger.debug("ClientHandler: Activate socket #{inspect(data.active_count)}") - HandlerHelpers.activate(data.sock) - end - + HandlerHelpers.activate(data.sock) 0 end end diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index 492aadf2..1d7ff475 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -295,8 +295,7 @@ defmodule Supavisor.DbHandler do Logger.debug("DbHandler: Got write replica message #{inspect(bin)}") if String.ends_with?(bin, Server.ready_for_query()) do - if data.active_count >= @switch_active_count, - do: HandlerHelpers.activate(data.sock) + HandlerHelpers.activate(data.sock) {_, stats} = Telem.network_usage(:db, data.sock, data.id, data.stats) @@ -363,7 +362,7 @@ defmodule Supavisor.DbHandler do end if state == :busy or data.mode == :session do - sock_send(data.sock, <>) + sock_send(data.sock, Server.terminate_message()) :gen_tcp.close(elem(data.sock, 1)) {:stop, {:client_handler_down, data.mode}} else diff --git a/lib/supavisor/protocol/server.ex b/lib/supavisor/protocol/server.ex index 02744356..9a865134 100644 --- a/lib/supavisor/protocol/server.ex +++ b/lib/supavisor/protocol/server.ex @@ -15,6 +15,7 @@ defmodule Supavisor.Protocol.Server do @scram_request <> @msg_cancel_header <<16::32, 1234::16, 5678::16>> @application_name <> + @terminate_message <> defmodule Pkt do @moduledoc "Representing a packet structure with tag, length, and payload fields." @@ -470,4 +471,7 @@ defmodule Supavisor.Protocol.Server do @spec application_name() :: binary def application_name, do: @application_name + + @spec terminate_message() :: binary + def terminate_message(), do: @terminate_message end From 43dd1fcad02ad9e77201cd4fcb6b6958bf5c994a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Fri, 15 Nov 2024 12:06:06 +0100 Subject: [PATCH 90/97] chore: remove unneeded Plugs (#484) These plugs weren't used at all, as we do not serve any static assets nor we need live reload for anything (as we do not have any form of web UI). --- lib/supavisor_web/endpoint.ex | 18 ------------------ mix.exs | 1 - mix.lock | 2 -- 3 files changed, 21 deletions(-) diff --git a/lib/supavisor_web/endpoint.ex b/lib/supavisor_web/endpoint.ex index 2f2d2b30..92f8efc6 100644 --- a/lib/supavisor_web/endpoint.ex +++ b/lib/supavisor_web/endpoint.ex @@ -12,24 +12,6 @@ defmodule SupavisorWeb.Endpoint do socket "/live", Phoenix.LiveView.Socket, websocket: [connect_info: [session: @session_options]] - # Serve at "/" the static files from "priv/static" directory. - # - # You should set gzip to true if you are running phx.digest - # when deploying your static files in production. - plug Plug.Static, - at: "/", - from: :supavisor, - gzip: false, - only: ~w(assets fonts images favicon-32x32.png robots.txt) - - # Code reloading can be explicitly enabled under the - # :code_reloader configuration of your endpoint. - if code_reloading? do - socket "/phoenix/live_reload/socket", Phoenix.LiveReloader.Socket - plug Phoenix.LiveReloader - plug Phoenix.CodeReloader - end - plug Phoenix.LiveDashboard.RequestLogger, param_key: "request_logger", cookie_key: "request_logger" diff --git a/mix.exs b/mix.exs index 4f3bc45f..b50573a0 100644 --- a/mix.exs +++ b/mix.exs @@ -43,7 +43,6 @@ defmodule Supavisor.MixProject do {:ecto_sql, "~> 3.10"}, {:postgrex, ">= 0.0.0"}, {:phoenix_view, "~> 2.0.2"}, - {:phoenix_live_reload, "~> 1.2", only: :dev}, {:phoenix_live_view, "~> 0.20.0"}, {:phoenix_live_dashboard, "~> 0.7"}, {:telemetry_poller, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index 63a3aa0c..cb9bf545 100644 --- a/mix.lock +++ b/mix.lock @@ -43,7 +43,6 @@ "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"}, "open_api_spex": {:hex, :open_api_spex, "3.21.2", "6a704f3777761feeb5657340250d6d7332c545755116ca98f33d4b875777e1e5", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "f42ae6ed668b895ebba3e02773cfb4b41050df26f803f2ef634c72a7687dc387"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.4.0", "63ca1742f92f00059298f478048dfb826f4b20d49534493d6919a0db39b6db04", [:mix, :rebar3], [], "hexpm", "3dfbbfaa2c2ed3121c5c483162836c4f9027def469c41578af5ef32589fcfc58"}, - "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, "peep": {:hex, :peep, "3.3.0", "ece8c38f0e3cfeecf8739d377c228c7e2b34d947f6b4817a183be37c88b94ebe", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:plug, "~> 1.16", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "19c84bf78c4eee97cb0df33d4ea628e93b6ab148ee19c289c499d1d62b3e78cb"}, "pg_types": {:hex, :pg_types, "0.4.0", "3ce365c92903c5bb59c0d56382d842c8c610c1b6f165e20c4b652c96fa7e9c14", [:rebar3], [], "hexpm", "b02efa785caececf9702c681c80a9ca12a39f9161a846ce17b01fb20aeeed7eb"}, "pgo": {:hex, :pgo, "0.14.0", "f53711d103d7565db6fc6061fcf4ff1007ab39892439be1bb02d9f686d7e6663", [:rebar3], [{:backoff, "~> 1.1.6", [hex: :backoff, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:pg_types, "~> 0.4.0", [hex: :pg_types, repo: "hexpm", optional: false]}], "hexpm", "71016c22599936e042dc0012ee4589d24c71427d266292f775ebf201d97df9c9"}, @@ -51,7 +50,6 @@ "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.3", "f686701b0499a07f2e3b122d84d52ff8a31f5def386e03706c916f6feddf69ef", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "909502956916a657a197f94cc1206d9a65247538de8a5e186f7537c895d95764"}, "phoenix_html": {:hex, :phoenix_html, "4.1.1", "4c064fd3873d12ebb1388425a8f2a19348cef56e7289e1998e2d2fa758aa982e", [:mix], [], "hexpm", "f2f2df5a72bc9a2f510b21497fd7d2b86d932ec0598f0210fed4114adc546c6f"}, "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.4", "4508e481f791ce62ec6a096e13b061387158cbeefacca68c6c1928e1305e23ed", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "2984aae96994fbc5c61795a73b8fb58153b41ff934019cfb522343d2d3817d59"}, - "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.5.3", "f2161c207fda0e4fb55165f650f7f8db23f02b29e3bff00ff7ef161d6ac1f09d", [:mix], [{:file_system, "~> 0.3 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "b4ec9cd73cb01ff1bd1cac92e045d13e7030330b74164297d1aee3907b54803c"}, "phoenix_live_view": {:hex, :phoenix_live_view, "0.20.17", "f396bbdaf4ba227b82251eb75ac0afa6b3da5e509bc0d030206374237dfc9450", [:mix], [{:floki, "~> 0.36", [hex: :floki, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a61d741ffb78c85fdbca0de084da6a48f8ceb5261a79165b5a0b59e5f65ce98b"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, From 32c0086d633d5805a083e47fce7eaf0e8d821f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Fri, 15 Nov 2024 12:24:26 +0100 Subject: [PATCH 91/97] test: add integration tests against Postgres.js (#479) --- .github/workflows/elixir.yml | 48 + docker-compose.db.yml | 2 +- docker-compose.yml | 2 +- flake.nix | 18 +- test/integration/external_test.exs | 125 + test/integration/js/.gitignore | 1 + test/integration/js/package.json | 13 + test/integration/js/postgres/copy.csv | 2 + test/integration/js/postgres/index.js | 2601 +++++++++++++++++ test/integration/js/postgres/select-param.sql | 1 + test/integration/js/postgres/select.sql | 1 + test/integration/js/postgres/test.js | 87 + test/integration/js/yarn.lock | 8 + test/test_helper.exs | 5 +- typos.toml | 6 + 15 files changed, 2916 insertions(+), 4 deletions(-) create mode 100644 test/integration/external_test.exs create mode 100644 test/integration/js/.gitignore create mode 100644 test/integration/js/package.json create mode 100644 test/integration/js/postgres/copy.csv create mode 100644 test/integration/js/postgres/index.js create mode 100644 test/integration/js/postgres/select-param.sql create mode 100644 test/integration/js/postgres/select.sql create mode 100644 test/integration/js/postgres/test.js create mode 100644 test/integration/js/yarn.lock diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index ae1c23cc..e900a00e 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -127,6 +127,54 @@ jobs: - name: Run tests run: mix test + integration: + name: Run integration tests + runs-on: u22-arm-runner + needs: [deps] + + steps: + - uses: actions/checkout@v4 + - name: Setup Elixir + id: beam + uses: erlef/setup-beam@v1 + with: + otp-version: '25.3.2.7' + elixir-version: '1.14.5' + - uses: actions/setup-node@v4 + with: + node-version: 'lts/*' + - name: Set up Rust + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: stable + - name: Cache Mix + uses: actions/cache@v4 + with: + path: deps + key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + - name: Cache native + uses: actions/cache@v4 + with: + path: | + _build/${{ env.MIX_ENV }}/lib/supavisor/native + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-build-native-${{ hashFiles(format('{0}{1}', github.workspace, '/native/**/Cargo.lock')) }} + restore-keys: | + ${{ runner.os }}-build-native- + - name: Compile deps + run: mix deps.compile + - name: Compile + run: mix compile + - name: Set up Postgres + run: docker-compose -f ./docker-compose.db.yml up -d + - name: Start epmd + run: epmd -daemon + - name: Run tests + run: mix test --only integration --trace + dialyzer: name: Dialyze runs-on: u22-arm-runner diff --git a/docker-compose.db.yml b/docker-compose.db.yml index 16bdf76a..00a8f5de 100644 --- a/docker-compose.db.yml +++ b/docker-compose.db.yml @@ -10,7 +10,7 @@ services: - ./dev/postgres:/docker-entrypoint-initdb.d/ # Uncomment to set MD5 authentication method on uninitialized databases # - ./dev/postgres/md5/etc/postgresql/pg_hba.conf:/etc/postgresql/pg_hba.conf - command: postgres -c config_file=/etc/postgresql/postgresql.conf + command: postgres -c config_file=/etc/postgresql/postgresql.conf -c max_prepared_transactions=2000 environment: POSTGRES_HOST: /var/run/postgresql POSTGRES_PASSWORD: postgres diff --git a/docker-compose.yml b/docker-compose.yml index 8c1eaf98..b09159f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - ./dev/postgres:/docker-entrypoint-initdb.d/ # Uncomment to set MD5 authentication method on uninitialized databases # - ./dev/postgres/md5/etc/postgresql/pg_hba.conf:/etc/postgresql/pg_hba.conf - command: postgres -c config_file=/etc/postgresql/postgresql.conf + command: postgres -c config_file=/etc/postgresql/postgresql.conf -c max_prepared_transactions=2000 environment: POSTGRES_HOST: /var/run/postgresql POSTGRES_PASSWORD: postgres diff --git a/flake.nix b/flake.nix index a7bfc0c0..6822cc68 100644 --- a/flake.nix +++ b/flake.nix @@ -53,7 +53,12 @@ { pre-commit.hooks = { alejandra.enable = true; - typos.enable = true; + typos = { + enable = true; + excludes = [ + "test/integration/" + ]; + }; }; } { @@ -78,6 +83,7 @@ services.postgres = { enable = true; + package = pkgs.postgresql_15; initialScript = '' ${builtins.readFile ./dev/postgres/00-setup.sql} @@ -85,6 +91,9 @@ ''; listen_addresses = "127.0.0.1"; port = 6432; + settings = { + max_prepared_transactions = 262143; + }; }; process.implementation = "honcho"; @@ -92,6 +101,13 @@ # Force connection through TCP instead of Unix socket env.PGHOST = lib.mkForce ""; } + { + languages.javascript = { + enable = true; + bun.enable = true; + yarn.enable = true; + }; + } ({ pkgs, lib, diff --git a/test/integration/external_test.exs b/test/integration/external_test.exs new file mode 100644 index 00000000..30299bc1 --- /dev/null +++ b/test/integration/external_test.exs @@ -0,0 +1,125 @@ +defmodule Supavisor.Integration.ExternalTest do + use ExUnit.Case, async: false + + @moduletag integration: true + + setup_all do + npm = + get_tool("yarn") || get_tool("npm") || get_tool("bun") || + raise "Cannot find neither Yarn nor NPM" + + assert {_, 0} = System.cmd(npm, ~w[install], cd: suite("js")) + + {:ok, npm: npm} + end + + setup :external_id + + setup ctx do + if get_tool(ctx.runtime) do + :ok + else + raise "Runtime not available" + end + end + + describe "Postgres.js" do + @describetag library: "postgres.js", suite: "js" + + @tag runtime: "node", mode: "session" + test "Node session", ctx do + assert_run(ctx, ~w[postgres/index.js]) + end + + @tag runtime: "node", mode: "transaction" + test "Node transaction", ctx do + assert_run(ctx, ~w[postgres/index.js]) + end + + # These currently do not pass + # @tag runtime: "bun", mode: "session" + # test "Bun session", ctx do + # assert_run ctx, ~w[postgres/index.js], suite: "js" + # end + # + # @tag runtime: "bun", mode: "transaction" + # test "Bun transaction", ctx do + # assert_run ctx, ~w[postgres/index.js], suite: "js" + # end + # + # @tag runtime: "deno", mode: "session" + # test "Deno session", ctx do + # assert_run ctx, ~w[run --allow-all postgres/index.js], suite: "js" + # end + # + # @tag runtime: "deno", mode: "transaction" + # test "Deno transaction", ctx do + # assert_run ctx, ~w[run --allow-all postgres/index.js], suite: "js" + # end + end + + defp assert_run(ctx, args, opts \\ []) do + suite = suite(ctx.suite) + + env = + [ + {"PGMODE", ctx.mode}, + {"PGDATABASE", ctx.db}, + {"PGHOST", "localhost"}, + {"PGPORT", to_string(port(ctx.mode))}, + {"PGUSER", ctx.user}, + {"PGPASS", "postgres"} + ] ++ (opts[:env] || []) + + assert {output, code} = + System.cmd(ctx.runtime, args, + env: env, + cd: suite, + stderr_to_stdout: true + ) + + assert code == 0, output + end + + ## UTILS + + defp suite(name), do: Path.join(__DIR__, name) + + defp get_tool(name), do: System.find_executable(name) + + defp port("session"), do: Application.fetch_env!(:supavisor, :proxy_port_session) + defp port("transaction"), do: Application.fetch_env!(:supavisor, :proxy_port_transaction) + + defp external_id(ctx) do + external_id = + [ctx.runtime, ctx.library, ctx.mode] + |> Enum.map_join("_", &String.replace(&1, ~r/\W/, "")) + + # Ensure that there are no leftovers + _ = Supavisor.Tenants.delete_tenant_by_external_id(external_id) + + _ = Supavisor.Repo.query("DROP DATABASE IF EXISTS #{external_id}") + assert {:ok, _} = Supavisor.Repo.query("CREATE DATABASE #{external_id}") + + assert {:ok, tenant} = + Supavisor.Tenants.create_tenant(%{ + default_parameter_status: %{}, + db_host: "localhost", + db_port: 6432, + db_database: external_id, + auth_query: "SELECT rolname, rolpassword FROM pg_authid WHERE rolname=$1;", + external_id: external_id, + users: [ + %{ + "pool_size" => 3, + "db_user" => "postgres", + "db_password" => "postgres", + "is_manager" => true, + "mode_type" => "session" + } + ] + }) + + {:ok, user: "postgres.#{external_id}", db: tenant.db_database, external_id: external_id} + end +end diff --git a/test/integration/js/.gitignore b/test/integration/js/.gitignore new file mode 100644 index 00000000..07e6e472 --- /dev/null +++ b/test/integration/js/.gitignore @@ -0,0 +1 @@ +/node_modules diff --git a/test/integration/js/package.json b/test/integration/js/package.json new file mode 100644 index 00000000..4a83eaaa --- /dev/null +++ b/test/integration/js/package.json @@ -0,0 +1,13 @@ +{ + "name": "supavisor-integration", + "version": "1.0.0", + "main": "index.js", + "type": "module", + "license": "MIT", + "scripts": { + "test:postgres": "node ./postgres/index.js" + }, + "dependencies": { + "postgres": "^3.4.5" + } +} diff --git a/test/integration/js/postgres/copy.csv b/test/integration/js/postgres/copy.csv new file mode 100644 index 00000000..6622044e --- /dev/null +++ b/test/integration/js/postgres/copy.csv @@ -0,0 +1,2 @@ +1 2 3 +4 5 6 diff --git a/test/integration/js/postgres/index.js b/test/integration/js/postgres/index.js new file mode 100644 index 00000000..a8f68216 --- /dev/null +++ b/test/integration/js/postgres/index.js @@ -0,0 +1,2601 @@ +import { t, nt, ot } from './test.js' // eslint-disable-line +import net from 'node:net' +import fs from 'node:fs' +import crypto from 'node:crypto' + +import postgres from 'postgres' +const delay = ms => new Promise(r => setTimeout(r, ms)) + +const rel = x => new URL(x, import.meta.url) +const idle_timeout = 1 + +const login = { + user: process.env.PGUSER, + pass: process.env.PGPASS, +} + +//const login_md5 = { +// user: 'postgres_js_test_md5', +// pass: 'postgres_js_test_md5' +//} +// +//const login_scram = { +// user: 'postgres_js_test_scram', +// pass: 'postgres_js_test_scram' +//} + +const options = { + host: process.env.PGHOST, + port: process.env.PGPORT, + db: process.env.PGDATABASE, + prepare: (process.env.PGMODE != 'transaction'), + user: login.user, + pass: login.pass, + idle_timeout, + connect_timeout: 1, + max: 1 +} + +const sql = postgres(options) + +await sql`DROP TABLE IF EXISTS test`; + +//t('Connects with no options', async() => { +// const sql = postgres({ max: 1 }) +// +// const result = (await sql`select 1 as x`)[0].x +// await sql.end() +// +// return [1, result] +//}) + +//t('Uses default database without slash', async() => { +// const sql = postgres('postgres://localhost') +// return [sql.options.user, sql.options.database] +//}) +// +//t('Uses default database with slash', async() => { +// const sql = postgres('postgres://localhost/') +// return [sql.options.user, sql.options.database] +//}) + +t('Result is array', async() => + [true, Array.isArray(await sql`select 1`)] +) + +t('Result has count', async() => + [1, (await sql`select 1`).count] +) + +t('Result has command', async() => + ['SELECT', (await sql`select 1`).command] +) + +t('Create table', async() => + ['CREATE TABLE', (await sql`create table test(int int)`).command, await sql`drop table test`] +) + +t('Drop table', { timeout: 2 }, async() => { + await sql`create table test(int int)` + return ['DROP TABLE', (await sql`drop table test`).command] +}) + +t('null', async() => + [null, (await sql`select ${ null } as x`)[0].x] +) + +t('Integer', async() => + ['1', (await sql`select ${ 1 } as x`)[0].x] +) + +t('String', async() => + ['hello', (await sql`select ${ 'hello' } as x`)[0].x] +) + +t('Boolean false', async() => + [false, (await sql`select ${ false } as x`)[0].x] +) + +t('Boolean true', async() => + [true, (await sql`select ${ true } as x`)[0].x] +) + +t('Date', async() => { + const now = new Date() + return [0, now - (await sql`select ${ now } as x`)[0].x] +}) + +t('Json', async() => { + const x = (await sql`select ${ sql.json({ a: 'hello', b: 42 }) } as x`)[0].x + return ['hello,42', [x.a, x.b].join()] +}) + +t('implicit json', async() => { + const x = (await sql`select ${ { a: 'hello', b: 42 } }::json as x`)[0].x + return ['hello,42', [x.a, x.b].join()] +}) + +t('implicit jsonb', async() => { + const x = (await sql`select ${ { a: 'hello', b: 42 } }::jsonb as x`)[0].x + return ['hello,42', [x.a, x.b].join()] +}) + +t('Empty array', async() => + [true, Array.isArray((await sql`select ${ sql.array([], 1009) } as x`)[0].x)] +) + +t('String array', async() => + ['123', (await sql`select ${ '{1,2,3}' }::int[] as x`)[0].x.join('')] +) + +t('Array of Integer', async() => + ['3', (await sql`select ${ sql.array([1, 2, 3]) } as x`)[0].x[2]] +) + +t('Array of String', async() => + ['c', (await sql`select ${ sql.array(['a', 'b', 'c']) } as x`)[0].x[2]] +) + +t('Array of Date', async() => { + const now = new Date() + return [now.getTime(), (await sql`select ${ sql.array([now, now, now]) } as x`)[0].x[2].getTime()] +}) + +t('Array of Box', async() => [ + '(3,4),(1,2);(6,7),(4,5)', + (await sql`select ${ '{(1,2),(3,4);(4,5),(6,7)}' }::box[] as x`)[0].x.join(';') +]) + +t('Nested array n2', async() => + ['4', (await sql`select ${ sql.array([[1, 2], [3, 4]]) } as x`)[0].x[1][1]] +) + +t('Nested array n3', async() => + ['6', (await sql`select ${ sql.array([[[1, 2]], [[3, 4]], [[5, 6]]]) } as x`)[0].x[2][0][1]] +) + +t('Escape in arrays', async() => + ['Hello "you",c:\\windows', (await sql`select ${ sql.array(['Hello "you"', 'c:\\windows']) } as x`)[0].x.join(',')] +) + +t('Escapes', async() => { + return ['hej"hej', Object.keys((await sql`select 1 as ${ sql('hej"hej') }`)[0])[0]] +}) + +t('null for int', async() => { + await sql`create table test (x int)` + return [1, (await sql`insert into test values(${ null })`).count, await sql`drop table test`] +}) + +t('Throws on illegal transactions', async() => { + const sql = postgres({ ...options, max: 2, fetch_types: false }) + const error = await sql`begin`.catch(e => e) + return [ + error.code, + 'UNSAFE_TRANSACTION' + ] +}) + +t('Transaction throws', async() => { + await sql`create table test (a int)` + return ['22P02', await sql.begin(async sql => { + await sql`insert into test values(1)` + await sql`insert into test values('hej')` + }).catch(x => x.code), await sql`drop table test`] +}) + +t('Transaction rolls back', async() => { + await sql`create table test (a int)` + await sql.begin(async sql => { + await sql`insert into test values(1)` + await sql`insert into test values('hej')` + }).catch(() => { /* ignore */ }) + return [0, (await sql`select a from test`).count, await sql`drop table test`] +}) + +t('Transaction throws on uncaught savepoint', async() => { + await sql`create table test (a int)` + + return ['fail', (await sql.begin(async sql => { + await sql`insert into test values(1)` + await sql.savepoint(async sql => { + await sql`insert into test values(2)` + throw new Error('fail') + }) + }).catch((err) => err.message)), await sql`drop table test`] +}) + +t('Transaction throws on uncaught named savepoint', async() => { + await sql`create table test (a int)` + + return ['fail', (await sql.begin(async sql => { + await sql`insert into test values(1)` + await sql.savepoit('watpoint', async sql => { + await sql`insert into test values(2)` + throw new Error('fail') + }) + }).catch(() => 'fail')), await sql`drop table test`] +}) + +t('Transaction succeeds on caught savepoint', async() => { + await sql`create table test (a int)` + await sql.begin(async sql => { + await sql`insert into test values(1)` + await sql.savepoint(async sql => { + await sql`insert into test values(2)` + throw new Error('please rollback') + }).catch(() => { /* ignore */ }) + await sql`insert into test values(3)` + }) + + return ['2', (await sql`select count(1) from test`)[0].count, await sql`drop table test`] +}) + +t('Savepoint returns Result', async() => { + let result + await sql.begin(async sql => { + result = await sql.savepoint(sql => + sql`select 1 as x` + ) + }) + + return [1, result[0].x] +}) + +if (options.prepare) { + t('Prepared transaction', async() => { + await sql`create table test (a int)` + + await sql.begin(async sql => { + await sql`insert into test values(1)` + await sql.prepare('tx1') + }) + + await sql`commit prepared 'tx1'` + + return ['1', (await sql`select count(1) from test`)[0].count, await sql`drop table test`] + }) +} + +t('Transaction requests are executed implicitly', async() => { + const sql = postgres({ ...options, debug: true, idle_timeout: 1, fetch_types: false }) + return [ + 'testing', + (await sql.begin(sql => [ + sql`select set_config('postgres_js.test', 'testing', true)`, + sql`select current_setting('postgres_js.test') as x` + ]))[1][0].x + ] +}) + +t('Uncaught transaction request errors bubbles to transaction', async() => [ + '42703', + (await sql.begin(sql => [ + sql`select wat`, + sql`select current_setting('postgres_js.test') as x, ${ 1 } as a` + ]).catch(e => e.code)) +]) + +t('Fragments in transactions', async() => [ + true, + (await sql.begin(sql => sql`select true as x where ${ sql`1=1` }`))[0].x +]) + +t('Transaction rejects with rethrown error', async() => [ + 'WAT', + await sql.begin(async sql => { + try { + await sql`select exception` + } catch (ex) { + throw new Error('WAT') + } + }).catch(e => e.message) +]) + +t('Parallel transactions', async() => { + await sql`create table test (a int)` + return ['11', (await Promise.all([ + sql.begin(sql => sql`select 1`), + sql.begin(sql => sql`select 1`) + ])).map(x => x.count).join(''), await sql`drop table test`] +}) + +t('Many transactions at beginning of connection', async() => { + const sql = postgres(options) + const xs = await Promise.all(Array.from({ length: 100 }, () => sql.begin(sql => sql`select 1`))) + return [100, xs.length] +}) + +t('Transactions array', async() => { + await sql`create table test (a int)` + + return ['11', (await sql.begin(sql => [ + sql`select 1`.then(x => x), + sql`select 1` + ])).map(x => x.count).join(''), await sql`drop table test`] +}) + +t('Transaction waits', async() => { + await sql`create table test (a int)` + await sql.begin(async sql => { + await sql`insert into test values(1)` + await sql.savepoint(async sql => { + await sql`insert into test values(2)` + throw new Error('please rollback') + }).catch(() => { /* ignore */ }) + await sql`insert into test values(3)` + }) + + return ['11', (await Promise.all([ + sql.begin(sql => sql`select 1`), + sql.begin(sql => sql`select 1`) + ])).map(x => x.count).join(''), await sql`drop table test`] +}) + +t('Helpers in Transaction', async() => { + return ['1', (await sql.begin(async sql => + await sql`select ${ sql({ x: 1 }) }` + ))[0].x] +}) + +t('Undefined values throws', async() => { + let error + + await sql` + select ${ undefined } as x + `.catch(x => error = x.code) + + return ['UNDEFINED_VALUE', error] +}) + +t('Transform undefined', async() => { + const sql = postgres({ ...options, transform: { undefined: null } }) + return [null, (await sql`select ${ undefined } as x`)[0].x] +}) + +t('Transform undefined in array', async() => { + const sql = postgres({ ...options, transform: { undefined: null } }) + return [null, (await sql`select * from (values ${ sql([undefined, undefined]) }) as x(x, y)`)[0].y] +}) + +t('Null sets to null', async() => + [null, (await sql`select ${ null } as x`)[0].x] +) + +t('Throw syntax error', async() => + ['42601', (await sql`wat 1`.catch(x => x)).code] +) + +t('Connect using uri', async() => + [true, await new Promise((resolve, reject) => { + const sql = postgres(`postgres://${login.user}:${login.pass}@${options.host}:${options.port}/${options.db}`, { + idle_timeout + }) + sql`select 1`.then(() => resolve(true), reject) + })] +) + +t('Options from uri with special characters in user and pass', async() => { + const opt = postgres({ user: 'öla', pass: 'pass^word' }).options + return [[opt.user, opt.pass].toString(), 'öla,pass^word'] +}) + +t('Fail with proper error on no host', async() => + ['ECONNREFUSED', (await new Promise((resolve, reject) => { + const sql = postgres('postgres://localhost:33333/' + options.db, { + idle_timeout + }) + sql`select 1`.then(reject, resolve) + })).code] +) + +// REASON: No SSL in local testing (so far) +//t('Connect using SSL', async() => +// [true, (await new Promise((resolve, reject) => { +// postgres({ +// ssl: { rejectUnauthorized: false }, +// idle_timeout +// })`select 1`.then(() => resolve(true), reject) +// }))] +//) +// +//t('Connect using SSL require', async() => +// [true, (await new Promise((resolve, reject) => { +// postgres({ +// ssl: 'require', +// idle_timeout +// })`select 1`.then(() => resolve(true), reject) +// }))] +//) +// +//t('Connect using SSL prefer', async() => { +// await exec('psql', ['-c', 'alter system set ssl=off']) +// await exec('psql', ['-c', 'select pg_reload_conf()']) +// +// const sql = postgres({ +// ssl: 'prefer', +// idle_timeout +// }) +// +// return [ +// 1, (await sql`select 1 as x`)[0].x, +// await exec('psql', ['-c', 'alter system set ssl=on']), +// await exec('psql', ['-c', 'select pg_reload_conf()']) +// ] +//}) +// +//t('Reconnect using SSL', { timeout: 2 }, async() => { +// const sql = postgres({ +// ssl: 'require', +// idle_timeout: 0.1 +// }) +// +// await sql`select 1` +// await delay(200) +// +// return [1, (await sql`select 1 as x`)[0].x] +//}) +// +//t('Login without password', async() => { +// return [true, (await postgres({ ...options, ...login })`select true as x`)[0].x] +//}) + +// Reason: No MD5 +//t('Login using MD5', async() => { +// return [true, (await postgres({ ...options, ...login_md5 })`select true as x`)[0].x] +//}) +// +//t('Login using scram-sha-256', async() => { +// return [true, (await postgres({ ...options, ...login_scram })`select true as x`)[0].x] +//}) + +// Reason: No tests for SCRAM (for now) +//t('Parallel connections using scram-sha-256', { +// timeout: 2 +//}, async() => { +// const sql = postgres({ ...options, ...login_scram }) +// return [true, (await Promise.all([ +// sql`select true as x, pg_sleep(0.01)`, +// sql`select true as x, pg_sleep(0.01)`, +// sql`select true as x, pg_sleep(0.01)` +// ]))[0][0].x] +//}) +// +//t('Support dynamic password function', async() => { +// return [true, (await postgres({ +// ...options, +// ...login_scram, +// pass: () => 'postgres_js_test_scram' +// })`select true as x`)[0].x] +//}) +// +//t('Support dynamic async password function', async() => { +// return [true, (await postgres({ +// ...options, +// ...login_scram, +// pass: () => Promise.resolve('postgres_js_test_scram') +// })`select true as x`)[0].x] +//}) + +t('Point type', async() => { + const sql = postgres({ + ...options, + types: { + point: { + to: 600, + from: [600], + serialize: ([x, y]) => '(' + x + ',' + y + ')', + parse: (x) => x.slice(1, -1).split(',').map(x => +x) + } + } + }) + + await sql`create table test (x point)` + await sql`insert into test (x) values (${ sql.types.point([10, 20]) })` + return [20, (await sql`select x from test`)[0].x[1], await sql`drop table test`] +}) + +t('Point type array', async() => { + const sql = postgres({ + ...options, + types: { + point: { + to: 600, + from: [600], + serialize: ([x, y]) => '(' + x + ',' + y + ')', + parse: (x) => x.slice(1, -1).split(',').map(x => +x) + } + } + }) + + await sql`create table test (x point[])` + await sql`insert into test (x) values (${ sql.array([sql.types.point([10, 20]), sql.types.point([20, 30])]) })` + return [30, (await sql`select x from test`)[0].x[1][1], await sql`drop table test`] +}) + +t('sql file', async() => + [1, (await sql.file(rel('select.sql')))[0].x] +) + +t('sql file has forEach', async() => { + let result + await sql + .file(rel('select.sql'), { cache: false }) + .forEach(({ x }) => result = x) + + return [1, result] +}) + +t('sql file throws', async() => + ['ENOENT', (await sql.file(rel('selectomondo.sql')).catch(x => x.code))] +) + +t('sql file cached', async() => { + await sql.file(rel('select.sql')) + await delay(20) + + return [1, (await sql.file(rel('select.sql')))[0].x] +}) + +t('Parameters in file', async() => { + const result = await sql.file( + rel('select-param.sql'), + ['hello'] + ) + return ['hello', result[0].x] +}) + +t('Connection ended promise', async() => { + const sql = postgres(options) + + await sql.end() + + return [undefined, await sql.end()] +}) + +t('Connection ended timeout', async() => { + const sql = postgres(options) + + await sql.end({ timeout: 10 }) + + return [undefined, await sql.end()] +}) + +t('Connection ended error', async() => { + const sql = postgres(options) + await sql.end() + return ['CONNECTION_ENDED', (await sql``.catch(x => x.code))] +}) + +t('Connection end does not cancel query', async() => { + const sql = postgres(options) + + const promise = sql`select 1 as x`.execute() + + await sql.end() + + return [1, (await promise)[0].x] +}) + +t('Connection destroyed', async() => { + const sql = postgres(options) + process.nextTick(() => sql.end({ timeout: 0 })) + return ['CONNECTION_DESTROYED', await sql``.catch(x => x.code)] +}) + +t('Connection destroyed with query before', async() => { + const sql = postgres(options) + , error = sql`select pg_sleep(0.2)`.catch(err => err.code) + + sql.end({ timeout: 0 }) + return ['CONNECTION_DESTROYED', await error] +}) + +t('transform column', async() => { + const sql = postgres({ + ...options, + transform: { column: x => x.split('').reverse().join('') } + }) + + await sql`create table test (hello_world int)` + await sql`insert into test values (1)` + return ['dlrow_olleh', Object.keys((await sql`select * from test`)[0])[0], await sql`drop table test`] +}) + +t('column toPascal', async() => { + const sql = postgres({ + ...options, + transform: { column: postgres.toPascal } + }) + + await sql`create table test (hello_world int)` + await sql`insert into test values (1)` + return ['HelloWorld', Object.keys((await sql`select * from test`)[0])[0], await sql`drop table test`] +}) + +t('column toCamel', async() => { + const sql = postgres({ + ...options, + transform: { column: postgres.toCamel } + }) + + await sql`create table test (hello_world int)` + await sql`insert into test values (1)` + return ['helloWorld', Object.keys((await sql`select * from test`)[0])[0], await sql`drop table test`] +}) + +t('column toKebab', async() => { + const sql = postgres({ + ...options, + transform: { column: postgres.toKebab } + }) + + await sql`create table test (hello_world int)` + await sql`insert into test values (1)` + return ['hello-world', Object.keys((await sql`select * from test`)[0])[0], await sql`drop table test`] +}) + +t('Transform nested json in arrays', async() => { + const sql = postgres({ + ...options, + transform: postgres.camel + }) + return ['aBcD', (await sql`select '[{"a_b":1},{"c_d":2}]'::jsonb as x`)[0].x.map(Object.keys).join('')] +}) + +t('Transform deeply nested json object in arrays', async() => { + const sql = postgres({ + ...options, + transform: postgres.camel + }) + return [ + 'childObj_deeplyNestedObj_grandchildObj', + (await sql` + select '[{"nested_obj": {"child_obj": 2, "deeply_nested_obj": {"grandchild_obj": 3}}}]'::jsonb as x + `)[0].x.map(x => { + let result + for (const key in x) + result = [...Object.keys(x[key]), ...Object.keys(x[key].deeplyNestedObj)] + return result + })[0] + .join('_') + ] +}) + +t('Transform deeply nested json array in arrays', async() => { + const sql = postgres({ + ...options, + transform: postgres.camel + }) + return [ + 'childArray_deeplyNestedArray_grandchildArray', + (await sql` + select '[{"nested_array": [{"child_array": 2, "deeply_nested_array": [{"grandchild_array":3}]}]}]'::jsonb AS x + `)[0].x.map((x) => { + let result + for (const key in x) + result = [...Object.keys(x[key][0]), ...Object.keys(x[key][0].deeplyNestedArray[0])] + return result + })[0] + .join('_') + ] +}) + +t('Bypass transform for json primitive', async() => { + const sql = postgres({ + ...options, + transform: postgres.camel + }) + + const x = ( + await sql`select 'null'::json as a, 'false'::json as b, '"a"'::json as c, '1'::json as d` + )[0] + + return [ + JSON.stringify({ a: null, b: false, c: 'a', d: 1 }), + JSON.stringify(x) + ] +}) + +t('Bypass transform for jsonb primitive', async() => { + const sql = postgres({ + ...options, + transform: postgres.camel + }) + + const x = ( + await sql`select 'null'::jsonb as a, 'false'::jsonb as b, '"a"'::jsonb as c, '1'::jsonb as d` + )[0] + + return [ + JSON.stringify({ a: null, b: false, c: 'a', d: 1 }), + JSON.stringify(x) + ] +}) + +t('unsafe', async() => { + await sql`create table test (x int)` + return [1, (await sql.unsafe('insert into test values ($1) returning *', [1]))[0].x, await sql`drop table test`] +}) + +t('unsafe simple', async() => { + return [1, (await sql.unsafe('select 1 as x'))[0].x] +}) + +t('unsafe simple includes columns', async() => { + return ['x', (await sql.unsafe('select 1 as x').values()).columns[0].name] +}) + +t('unsafe describe', async() => { + const q = 'insert into test values (1)' + await sql`create table test(a int unique)` + await sql.unsafe(q).describe() + const x = await sql.unsafe(q).describe() + return [ + q, + x.string, + await sql`drop table test` + ] +}) + +t('simple query using unsafe with multiple statements', async() => { + return [ + '1,2', + (await sql.unsafe('select 1 as x;select 2 as x')).map(x => x[0].x).join() + ] +}) + +t('simple query using simple() with multiple statements', async() => { + return [ + '1,2', + (await sql`select 1 as x;select 2 as x`.simple()).map(x => x[0].x).join() + ] +}) + +if (options.prepare) { + t('listen and notify', async() => { + const sql = postgres(options) + const channel = 'hello' + const result = await new Promise(async r => { + await sql.listen(channel, r) + sql.notify(channel, 'works') + }) + + return [ + 'works', + result, + sql.end() + ] + }) + + t('double listen', async() => { + const sql = postgres(options) + , channel = 'hello' + + let count = 0 + + await new Promise((resolve, reject) => + sql.listen(channel, resolve) + .then(() => sql.notify(channel, 'world')) + .catch(reject) + ).then(() => count++) + + await new Promise((resolve, reject) => + sql.listen(channel, resolve) + .then(() => sql.notify(channel, 'world')) + .catch(reject) + ).then(() => count++) + + // for coverage + sql.listen('weee', () => { /* noop */ }).then(sql.end) + + return [2, count] + }) + +// Reason: No LISTEN/NOTIFY +//t('multiple listeners work after a reconnect', async() => { +// const sql = postgres(options) +// , xs = [] +// +// const s1 = await sql.listen('test', x => xs.push('1', x)) +// await sql.listen('test', x => xs.push('2', x)) +// await sql.notify('test', 'a') +// await delay(50) +// await sql`select pg_terminate_backend(${ s1.state.pid })` +// await delay(200) +// await sql.notify('test', 'b') +// await delay(50) +// sql.end() +// +// return ['1a2a1b2b', xs.join('')] +//}) +// +//t('listen and notify with weird name', async() => { +// const sql = postgres(options) +// const channel = 'wat-;.ø.§' +// const result = await new Promise(async r => { +// const { unlisten } = await sql.listen(channel, r) +// sql.notify(channel, 'works') +// await delay(50) +// await unlisten() +// }) +// +// return [ +// 'works', +// result, +// sql.end() +// ] +//}) +// +//t('listen and notify with upper case', async() => { +// const sql = postgres(options) +// const channel = 'withUpperChar' +// const result = await new Promise(async r => { +// await sql.listen(channel, r) +// sql.notify(channel, 'works') +// }) +// +// return [ +// 'works', +// result, +// sql.end() +// ] +//}) +// +//t('listen reconnects', { timeout: 2 }, async() => { +// const sql = postgres(options) +// , resolvers = {} +// , a = new Promise(r => resolvers.a = r) +// , b = new Promise(r => resolvers.b = r) +// +// let connects = 0 +// +// const { state: { pid } } = await sql.listen( +// 'test', +// x => x in resolvers && resolvers[x](), +// () => connects++ +// ) +// await sql.notify('test', 'a') +// await a +// await sql`select pg_terminate_backend(${ pid })` +// await delay(100) +// await sql.notify('test', 'b') +// await b +// sql.end() +// return [connects, 2] +//}) +// +//t('listen result reports correct connection state after reconnection', async() => { +// const sql = postgres(options) +// , xs = [] +// +// const result = await sql.listen('test', x => xs.push(x)) +// const initialPid = result.state.pid +// await sql.notify('test', 'a') +// await sql`select pg_terminate_backend(${ initialPid })` +// await delay(50) +// sql.end() +// +// return [result.state.pid !== initialPid, true] +//}) +// +//t('unlisten removes subscription', async() => { +// const sql = postgres(options) +// , xs = [] +// +// const { unlisten } = await sql.listen('test', x => xs.push(x)) +// await sql.notify('test', 'a') +// await delay(50) +// await unlisten() +// await sql.notify('test', 'b') +// await delay(50) +// sql.end() +// +// return ['a', xs.join('')] +//}) +// +//t('listen after unlisten', async() => { +// const sql = postgres(options) +// , xs = [] +// +// const { unlisten } = await sql.listen('test', x => xs.push(x)) +// await sql.notify('test', 'a') +// await delay(50) +// await unlisten() +// await sql.notify('test', 'b') +// await delay(50) +// await sql.listen('test', x => xs.push(x)) +// await sql.notify('test', 'c') +// await delay(50) +// sql.end() +// +// return ['ac', xs.join('')] +//}) +// +//t('multiple listeners and unlisten one', async() => { +// const sql = postgres(options) +// , xs = [] +// +// await sql.listen('test', x => xs.push('1', x)) +// const s2 = await sql.listen('test', x => xs.push('2', x)) +// await sql.notify('test', 'a') +// await delay(50) +// await s2.unlisten() +// await sql.notify('test', 'b') +// await delay(50) +// sql.end() +// +// return ['1a2a1b', xs.join('')] +//}) +} + +// Reason: We alter these parameters for PSQL, so it will not work as expected +//t('responds with server parameters (application_name)', async() => +// ['postgres.js', await new Promise((resolve, reject) => postgres({ +// ...options, +// onparameter: (k, v) => k === 'application_name' && resolve(v) +// })`select 1`.catch(reject))] +//) +// +//t('has server parameters', async() => { +// return ['postgres.js', (await sql`select 1`.then(() => sql.parameters.application_name))] +//}) + +t('big query body', { timeout: 2 }, async() => { + const size = 50000 + await sql`create table test (x int)` + return [size, (await sql`insert into test ${ + sql([...Array(size).keys()].map(x => ({ x }))) + }`).count, await sql`drop table test`] +}) + +t('Throws if more than 65534 parameters', async() => { + await sql`create table test (x int)` + return ['MAX_PARAMETERS_EXCEEDED', (await sql`insert into test ${ + sql([...Array(65535).keys()].map(x => ({ x }))) + }`.catch(e => e.code)), await sql`drop table test`] +}) + +t('let postgres do implicit cast of unknown types', async() => { + await sql`create table test (x timestamp with time zone)` + const [{ x }] = await sql`insert into test values (${ new Date().toISOString() }) returning *` + return [true, x instanceof Date, await sql`drop table test`] +}) + +t('only allows one statement', async() => + ['42601', await sql`select 1; select 2`.catch(e => e.code)] +) + +t('await sql() throws not tagged error', async() => { + let error + try { + await sql('select 1') + } catch (e) { + error = e.code + } + return ['NOT_TAGGED_CALL', error] +}) + +t('sql().then throws not tagged error', async() => { + let error + try { + sql('select 1').then(() => { /* noop */ }) + } catch (e) { + error = e.code + } + return ['NOT_TAGGED_CALL', error] +}) + +t('sql().catch throws not tagged error', async() => { + let error + try { + await sql('select 1') + } catch (e) { + error = e.code + } + return ['NOT_TAGGED_CALL', error] +}) + +t('sql().finally throws not tagged error', async() => { + let error + try { + sql('select 1').finally(() => { /* noop */ }) + } catch (e) { + error = e.code + } + return ['NOT_TAGGED_CALL', error] +}) + +t('little bobby tables', async() => { + const name = 'Robert\'); DROP TABLE students;--' + + await sql`create table students (name text, age int)` + await sql`insert into students (name) values (${ name })` + + return [ + name, (await sql`select name from students`)[0].name, + await sql`drop table students` + ] +}) + +t('Connection errors are caught using begin()', { + timeout: 2 +}, async() => { + let error + try { + const sql = postgres({ host: 'localhost', port: 1 }) + + await sql.begin(async(sql) => { + await sql`insert into test (label, value) values (${1}, ${2})` + }) + } catch (err) { + error = err + } + + return [ + true, + error.code === 'ECONNREFUSED' || + error.message === 'Connection refused (os error 61)' + ] +}) + +t('dynamic table name', async() => { + await sql`create table test(a int)` + return [ + 0, (await sql`select * from ${ sql('test') }`).count, + await sql`drop table test` + ] +}) + +t('dynamic schema name', async() => { + await sql`create table test(a int)` + return [ + 0, (await sql`select * from ${ sql('public') }.test`).count, + await sql`drop table test` + ] +}) + +t('dynamic schema and table name', async() => { + await sql`create table test(a int)` + return [ + 0, (await sql`select * from ${ sql('public.test') }`).count, + await sql`drop table test` + ] +}) + +t('dynamic column name', async() => { + return ['!not_valid', Object.keys((await sql`select 1 as ${ sql('!not_valid') }`)[0])[0]] +}) + +t('dynamic select as', async() => { + return ['2', (await sql`select ${ sql({ a: 1, b: 2 }) }`)[0].b] +}) + +t('dynamic select as pluck', async() => { + return [undefined, (await sql`select ${ sql({ a: 1, b: 2 }, 'a') }`)[0].b] +}) + +t('dynamic insert', async() => { + await sql`create table test (a int, b text)` + const x = { a: 42, b: 'the answer' } + + return ['the answer', (await sql`insert into test ${ sql(x) } returning *`)[0].b, await sql`drop table test`] +}) + +t('dynamic insert pluck', async() => { + await sql`create table test (a int, b text)` + const x = { a: 42, b: 'the answer' } + + return [null, (await sql`insert into test ${ sql(x, 'a') } returning *`)[0].b, await sql`drop table test`] +}) + +t('dynamic in with empty array', async() => { + await sql`create table test (a int)` + await sql`insert into test values (1)` + return [ + (await sql`select * from test where null in ${ sql([]) }`).count, + 0, + await sql`drop table test` + ] +}) + +t('dynamic in after insert', async() => { + await sql`create table test (a int, b text)` + const [{ x }] = await sql` + with x as ( + insert into test values (1, 'hej') + returning * + ) + select 1 in ${ sql([1, 2, 3]) } as x from x + ` + return [ + true, x, + await sql`drop table test` + ] +}) + +t('array insert', async() => { + await sql`create table test (a int, b int)` + return [2, (await sql`insert into test (a, b) values ${ sql([1, 2]) } returning *`)[0].b, await sql`drop table test`] +}) + +t('where parameters in()', async() => { + await sql`create table test (x text)` + await sql`insert into test values ('a')` + return [ + (await sql`select * from test where x in ${ sql(['a', 'b', 'c']) }`)[0].x, + 'a', + await sql`drop table test` + ] +}) + +t('where parameters in() values before', async() => { + return [2, (await sql` + with rows as ( + select * from (values (1), (2), (3), (4)) as x(a) + ) + select * from rows where a in ${ sql([3, 4]) } + `).count] +}) + +t('dynamic multi row insert', async() => { + await sql`create table test (a int, b text)` + const x = { a: 42, b: 'the answer' } + + return [ + 'the answer', + (await sql`insert into test ${ sql([x, x]) } returning *`)[1].b, await sql`drop table test` + ] +}) + +t('dynamic update', async() => { + await sql`create table test (a int, b text)` + await sql`insert into test (a, b) values (17, 'wrong')` + + return [ + 'the answer', + (await sql`update test set ${ sql({ a: 42, b: 'the answer' }) } returning *`)[0].b, await sql`drop table test` + ] +}) + +t('dynamic update pluck', async() => { + await sql`create table test (a int, b text)` + await sql`insert into test (a, b) values (17, 'wrong')` + + return [ + 'wrong', + (await sql`update test set ${ sql({ a: 42, b: 'the answer' }, 'a') } returning *`)[0].b, await sql`drop table test` + ] +}) + +t('dynamic select array', async() => { + await sql`create table test (a int, b text)` + await sql`insert into test (a, b) values (42, 'yay')` + return ['yay', (await sql`select ${ sql(['a', 'b']) } from test`)[0].b, await sql`drop table test`] +}) + +t('dynamic returning array', async() => { + await sql`create table test (a int, b text)` + return [ + 'yay', + (await sql`insert into test (a, b) values (42, 'yay') returning ${ sql(['a', 'b']) }`)[0].b, + await sql`drop table test` + ] +}) + +t('dynamic select args', async() => { + await sql`create table test (a int, b text)` + await sql`insert into test (a, b) values (42, 'yay')` + return ['yay', (await sql`select ${ sql('a', 'b') } from test`)[0].b, await sql`drop table test`] +}) + +t('dynamic values single row', async() => { + const [{ b }] = await sql` + select * from (values ${ sql(['a', 'b', 'c']) }) as x(a, b, c) + ` + + return ['b', b] +}) + +t('dynamic values multi row', async() => { + const [, { b }] = await sql` + select * from (values ${ sql([['a', 'b', 'c'], ['a', 'b', 'c']]) }) as x(a, b, c) + ` + + return ['b', b] +}) + +// Reason: we do not support custom connection parameters +//t('connection parameters', async() => { +// const sql = postgres({ +// ...options, +// connection: { +// 'some.var': 'yay' +// } +// }) +// +// return ['yay', (await sql`select current_setting('some.var') as x`)[0].x] +//}) + +t('Multiple queries', async() => { + return [4, (await Promise.all([ + sql`select 1`, + sql`select 2`, + sql`select 3`, + sql`select 4` + ])).length] +}) + +t('Multiple statements', async() => + [2, await sql.unsafe(` + select 1 as x; + select 2 as a; + `).then(([, [x]]) => x.a)] +) + +// Reason: We return different error, so that test will not work +//t('throws correct error when authentication fails', async() => { +// const sql = postgres({ +// ...options, +// pass: 'wrong' +// }) +// return ['28P01', await sql`select 1`.catch(e => e.code)] +//}) + +t('notice', async() => { + let notice + const log = console.log // eslint-disable-line + console.log = function(x) { // eslint-disable-line + notice = x + } + + await sql`create table if not exists users()` + await sql`create table if not exists users()` + + console.log = log // eslint-disable-line + + return ['NOTICE', notice.severity] +}) + +t('notice hook', async() => { + let notice + const sql = postgres({ + ...options, + onnotice: x => notice = x + }) + + await sql`create table if not exists users()` + await sql`create table if not exists users()` + + return ['NOTICE', notice.severity] +}) + +t('bytea serializes and parses', async() => { + const buf = Buffer.from('wat') + + await sql`create table test (x bytea)` + await sql`insert into test values (${ buf })` + + return [ + buf.toString(), + (await sql`select x from test`)[0].x.toString(), + await sql`drop table test` + ] +}) + +t('forEach', async() => { + let result + await sql`select 1 as x`.forEach(({ x }) => result = x) + return [1, result] +}) + +t('forEach returns empty array', async() => { + return [0, (await sql`select 1 as x`.forEach(() => { /* noop */ })).length] +}) + +t('Cursor', async() => { + const order = [] + await sql`select 1 as x union select 2 as x`.cursor(async([x]) => { + order.push(x.x + 'a') + await delay(100) + order.push(x.x + 'b') + }) + return ['1a1b2a2b', order.join('')] +}) + +t('Unsafe cursor', async() => { + const order = [] + await sql.unsafe('select 1 as x union select 2 as x').cursor(async([x]) => { + order.push(x.x + 'a') + await delay(100) + order.push(x.x + 'b') + }) + return ['1a1b2a2b', order.join('')] +}) + +t('Cursor custom n', async() => { + const order = [] + await sql`select * from generate_series(1,20)`.cursor(10, async(x) => { + order.push(x.length) + }) + return ['10,10', order.join(',')] +}) + +t('Cursor custom with rest n', async() => { + const order = [] + await sql`select * from generate_series(1,20)`.cursor(11, async(x) => { + order.push(x.length) + }) + return ['11,9', order.join(',')] +}) + +t('Cursor custom with less results than batch size', async() => { + const order = [] + await sql`select * from generate_series(1,20)`.cursor(21, async(x) => { + order.push(x.length) + }) + return ['20', order.join(',')] +}) + +t('Cursor cancel', async() => { + let result + await sql`select * from generate_series(1,10) as x`.cursor(async([{ x }]) => { + result = x + return sql.CLOSE + }) + return [1, result] +}) + +t('Cursor throw', async() => { + const order = [] + await sql`select 1 as x union select 2 as x`.cursor(async([x]) => { + order.push(x.x + 'a') + await delay(100) + throw new Error('watty') + }).catch(() => order.push('err')) + return ['1aerr', order.join('')] +}) + +t('Cursor error', async() => [ + '42601', + await sql`wat`.cursor(() => { /* noop */ }).catch((err) => err.code) +]) + +t('Multiple Cursors', { timeout: 2 }, async() => { + const result = [] + await sql.begin(async sql => [ + await sql`select 1 as cursor, x from generate_series(1,4) as x`.cursor(async([row]) => { + result.push(row.x) + await new Promise(r => setTimeout(r, 20)) + }), + await sql`select 2 as cursor, x from generate_series(101,104) as x`.cursor(async([row]) => { + result.push(row.x) + await new Promise(r => setTimeout(r, 10)) + }) + ]) + + return ['1,2,3,4,101,102,103,104', result.join(',')] +}) + +t('Cursor as async iterator', async() => { + const order = [] + for await (const [x] of sql`select generate_series(1,2) as x;`.cursor()) { + order.push(x.x + 'a') + await delay(10) + order.push(x.x + 'b') + } + + return ['1a1b2a2b', order.join('')] +}) + +t('Cursor as async iterator with break', async() => { + const order = [] + for await (const xs of sql`select generate_series(1,2) as x;`.cursor()) { + order.push(xs[0].x + 'a') + await delay(10) + order.push(xs[0].x + 'b') + break + } + + return ['1a1b', order.join('')] +}) + +t('Async Iterator Unsafe cursor', async() => { + const order = [] + for await (const [x] of sql.unsafe('select 1 as x union select 2 as x').cursor()) { + order.push(x.x + 'a') + await delay(10) + order.push(x.x + 'b') + } + return ['1a1b2a2b', order.join('')] +}) + +t('Async Iterator Cursor custom n', async() => { + const order = [] + for await (const x of sql`select * from generate_series(1,20)`.cursor(10)) + order.push(x.length) + + return ['10,10', order.join(',')] +}) + +t('Async Iterator Cursor custom with rest n', async() => { + const order = [] + for await (const x of sql`select * from generate_series(1,20)`.cursor(11)) + order.push(x.length) + + return ['11,9', order.join(',')] +}) + +t('Async Iterator Cursor custom with less results than batch size', async() => { + const order = [] + for await (const x of sql`select * from generate_series(1,20)`.cursor(21)) + order.push(x.length) + return ['20', order.join(',')] +}) + +t('Transform row', async() => { + const sql = postgres({ + ...options, + transform: { row: () => 1 } + }) + + return [1, (await sql`select 'wat'`)[0]] +}) + +t('Transform row forEach', async() => { + let result + const sql = postgres({ + ...options, + transform: { row: () => 1 } + }) + + await sql`select 1`.forEach(x => result = x) + + return [1, result] +}) + +t('Transform value', async() => { + const sql = postgres({ + ...options, + transform: { value: () => 1 } + }) + + return [1, (await sql`select 'wat' as x`)[0].x] +}) + +t('Transform columns from', async() => { + const sql = postgres({ + ...options, + transform: postgres.fromCamel + }) + await sql`create table test (a_test int, b_test text)` + await sql`insert into test ${ sql([{ aTest: 1, bTest: 1 }]) }` + await sql`update test set ${ sql({ aTest: 2, bTest: 2 }) }` + return [ + 2, + (await sql`select ${ sql('aTest', 'bTest') } from test`)[0].a_test, + await sql`drop table test` + ] +}) + +t('Transform columns to', async() => { + const sql = postgres({ + ...options, + transform: postgres.toCamel + }) + await sql`create table test (a_test int, b_test text)` + await sql`insert into test ${ sql([{ a_test: 1, b_test: 1 }]) }` + await sql`update test set ${ sql({ a_test: 2, b_test: 2 }) }` + return [ + 2, + (await sql`select a_test, b_test from test`)[0].aTest, + await sql`drop table test` + ] +}) + +t('Transform columns from and to', async() => { + const sql = postgres({ + ...options, + transform: postgres.camel + }) + await sql`create table test (a_test int, b_test text)` + await sql`insert into test ${ sql([{ aTest: 1, bTest: 1 }]) }` + await sql`update test set ${ sql({ aTest: 2, bTest: 2 }) }` + return [ + 2, + (await sql`select ${ sql('aTest', 'bTest') } from test`)[0].aTest, + await sql`drop table test` + ] +}) + +t('Transform columns from and to (legacy)', async() => { + const sql = postgres({ + ...options, + transform: { + column: { + to: postgres.fromCamel, + from: postgres.toCamel + } + } + }) + await sql`create table test (a_test int, b_test text)` + await sql`insert into test ${ sql([{ aTest: 1, bTest: 1 }]) }` + await sql`update test set ${ sql({ aTest: 2, bTest: 2 }) }` + return [ + 2, + (await sql`select ${ sql('aTest', 'bTest') } from test`)[0].aTest, + await sql`drop table test` + ] +}) + +// Reason: we do not support Unix sockets +//t('Unix socket', async() => { +// const sql = postgres({ +// ...options, +// host: process.env.PGSOCKET || '/tmp' // eslint-disable-line +// }) +// +// return [1, (await sql`select 1 as x`)[0].x] +//}) + +t('Big result', async() => { + return [100000, (await sql`select * from generate_series(1, 100000)`).count] +}) + +t('Debug', async() => { + let result + const sql = postgres({ + ...options, + debug: (connection_id, str) => result = str + }) + + await sql`select 1` + + return ['select 1', result] +}) + +t('bigint is returned as String', async() => [ + 'string', + typeof (await sql`select 9223372036854777 as x`)[0].x +]) + +t('int is returned as Number', async() => [ + 'number', + typeof (await sql`select 123 as x`)[0].x +]) + +t('numeric is returned as string', async() => [ + 'string', + typeof (await sql`select 1.2 as x`)[0].x +]) + +t('Async stack trace', async() => { + const sql = postgres({ ...options, debug: false }) + return [ + parseInt(new Error().stack.split('\n')[1].match(':([0-9]+):')[1]) + 1, + parseInt(await sql`error`.catch(x => x.stack.split('\n').pop().match(':([0-9]+):')[1])) + ] +}) + +t('Debug has long async stack trace', async() => { + const sql = postgres({ ...options, debug: true }) + + return [ + 'watyo', + await yo().catch(x => x.stack.match(/wat|yo/g).join('')) + ] + + function yo() { + return wat() + } + + function wat() { + return sql`error` + } +}) + +t('Error contains query string', async() => [ + 'selec 1', + (await sql`selec 1`.catch(err => err.query)) +]) + +t('Error contains query serialized parameters', async() => [ + 1, + (await sql`selec ${ 1 }`.catch(err => err.parameters[0])) +]) + +t('Error contains query raw parameters', async() => [ + 1, + (await sql`selec ${ 1 }`.catch(err => err.args[0])) +]) + +t('Query and parameters on errorare not enumerable if debug is not set', async() => { + const sql = postgres({ ...options, debug: false }) + + return [ + false, + (await sql`selec ${ 1 }`.catch(err => err.propertyIsEnumerable('parameters') || err.propertyIsEnumerable('query'))) + ] +}) + +t('Query and parameters are enumerable if debug is set', async() => { + const sql = postgres({ ...options, debug: true }) + + return [ + true, + (await sql`selec ${ 1 }`.catch(err => err.propertyIsEnumerable('parameters') && err.propertyIsEnumerable('query'))) + ] +}) + +t('connect_timeout', { timeout: 20 }, async() => { + const connect_timeout = 0.2 + const server = net.createServer() + server.listen() + const sql = postgres({ port: server.address().port, host: '127.0.0.1', connect_timeout }) + const start = Date.now() + let end + await sql`select 1`.catch((e) => { + if (e.code !== 'CONNECT_TIMEOUT') + throw e + end = Date.now() + }) + server.close() + return [connect_timeout, Math.floor((end - start) / 100) / 10] +}) + +t('connect_timeout throws proper error', async() => [ + 'CONNECT_TIMEOUT', + await postgres({ + ...options, + connect_timeout: 0.001 + })`select 1`.catch(e => e.code) +]) + +t('connect_timeout error message includes host:port', { timeout: 20 }, async() => { + const connect_timeout = 0.2 + const server = net.createServer() + server.listen() + const sql = postgres({ port: server.address().port, host: '127.0.0.1', connect_timeout }) + const port = server.address().port + let err + await sql`select 1`.catch((e) => { + if (e.code !== 'CONNECT_TIMEOUT') + throw e + err = e.message + }) + server.close() + return [['write CONNECT_TIMEOUT 127.0.0.1:', port].join(''), err] +}) + +t('requests works after single connect_timeout', async() => { + let first = true + + const sql = postgres({ + ...options, + connect_timeout: { valueOf() { return first ? (first = false, 0.0001) : 1 } } + }) + + return [ + 'CONNECT_TIMEOUT,,1', + [ + await sql`select 1 as x`.then(() => 'success', x => x.code), + await delay(10), + (await sql`select 1 as x`)[0].x + ].join(',') + ] +}) + +t('Postgres errors are of type PostgresError', async() => + [true, (await sql`bad keyword`.catch(e => e)) instanceof sql.PostgresError] +) + +t('Result has columns spec', async() => + ['x', (await sql`select 1 as x`).columns[0].name] +) + +t('forEach has result as second argument', async() => { + let x + await sql`select 1 as x`.forEach((_, result) => x = result) + return ['x', x.columns[0].name] +}) + +t('Result as arrays', async() => { + const sql = postgres({ + ...options, + transform: { + row: x => Object.values(x) + } + }) + + return ['1,2', (await sql`select 1 as a, 2 as b`)[0].join(',')] +}) + +t('Insert empty array', async() => { + await sql`create table tester (ints int[])` + return [ + Array.isArray((await sql`insert into tester (ints) values (${ sql.array([]) }) returning *`)[0].ints), + true, + await sql`drop table tester` + ] +}) + +t('Insert array in sql()', async() => { + await sql`create table tester (ints int[])` + return [ + Array.isArray((await sql`insert into tester ${ sql({ ints: sql.array([]) }) } returning *`)[0].ints), + true, + await sql`drop table tester` + ] +}) + +if (options.prepare) { + t('Automatically creates prepared statements', async() => { + const result = await sql`select * from pg_prepared_statements` + return [true, result.some(x => x.name = result.statement.name)] + }) + + t('no_prepare: true disables prepared statements (deprecated)', async() => { + const sql = postgres({ ...options, no_prepare: true }) + const result = await sql`select * from pg_prepared_statements` + return [false, result.some(x => x.name = result.statement.name)] + }) + + t('prepare: false disables prepared statements', async() => { + const sql = postgres({ ...options, prepare: false }) + const result = await sql`select * from pg_prepared_statements` + return [false, result.some(x => x.name = result.statement.name)] + }) + + t('prepare: true enables prepared statements', async() => { + const sql = postgres({ ...options, prepare: true }) + const result = await sql`select * from pg_prepared_statements` + return [true, result.some(x => x.name = result.statement.name)] + }) + + t('prepares unsafe query when "prepare" option is true', async() => { + const sql = postgres({ ...options, prepare: true }) + const result = await sql.unsafe('select * from pg_prepared_statements where name <> $1', ['bla'], { prepare: true }) + return [true, result.some(x => x.name = result.statement.name)] + }) + + t('does not prepare unsafe query by default', async() => { + const sql = postgres({ ...options, prepare: true }) + const result = await sql.unsafe('select * from pg_prepared_statements where name <> $1', ['bla']) + return [false, result.some(x => x.name = result.statement.name)] + }) + + t('Recreate prepared statements on transformAssignedExpr error', { timeout: 1 }, async() => { + const insert = () => sql`insert into test (name) values (${ '1' }) returning name` + await sql`create table test (name text)` + await insert() + await sql`alter table test alter column name type int using name::integer` + return [ + 1, + (await insert())[0].name, + await sql`drop table test` + ] + }) +} + +t('Throws correct error when retrying in transactions', async() => { + await sql`create table test(x int)` + const error = await sql.begin(sql => sql`insert into test (x) values (${ false })`).catch(e => e) + return [ + error.code, + '42804', + sql`drop table test` + ] +}) + +t('Recreate prepared statements on RevalidateCachedQuery error', async() => { + const select = () => sql`select name from test` + await sql`create table test (name text)` + await sql`insert into test values ('1')` + await select() + await sql`alter table test alter column name type int using name::integer` + return [ + 1, + (await select())[0].name, + await sql`drop table test` + ] +}) + +t('Properly throws routine error on not prepared statements', async() => { + await sql`create table x (x text[])` + const { routine } = await sql.unsafe(` + insert into x(x) values (('a', 'b')) + `).catch(e => e) + + return ['transformAssignedExpr', routine, await sql`drop table x`] +}) + +t('Properly throws routine error on not prepared statements in transaction', async() => { + const { routine } = await sql.begin(sql => [ + sql`create table x (x text[])`, + sql`insert into x(x) values (('a', 'b'))` + ]).catch(e => e) + + return ['transformAssignedExpr', routine] +}) + +t('Properly throws routine error on not prepared statements using file', async() => { + const { routine } = await sql.unsafe(` + create table x (x text[]); + insert into x(x) values (('a', 'b')); + `, { prepare: true }).catch(e => e) + + return ['transformAssignedExpr', routine] +}) + +t('Catches connection config errors', async() => { + const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) + + return [ + 'wat', + await sql`select 1`.catch((e) => e.message) + ] +}) + +t('Catches connection config errors with end', async() => { + const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) + + return [ + 'wat', + await sql`select 1`.catch((e) => e.message), + await sql.end() + ] +}) + +t('Catches query format errors', async() => [ + 'wat', + await sql.unsafe({ toString: () => { throw new Error('wat') } }).catch((e) => e.message) +]) + +// Reason: single host only +//t('Multiple hosts', { +// timeout: 1 +//}, async() => { +// const s1 = postgres({ idle_timeout }) +// , s2 = postgres({ idle_timeout, port: 5433 }) +// , sql = postgres('postgres://localhost:5432,localhost:5433', { idle_timeout, max: 1 }) +// , result = [] +// +// const id1 = (await s1`select system_identifier as x from pg_control_system()`)[0].x +// const id2 = (await s2`select system_identifier as x from pg_control_system()`)[0].x +// +// const x1 = await sql`select 1` +// result.push((await sql`select system_identifier as x from pg_control_system()`)[0].x) +// await s1`select pg_terminate_backend(${ x1.state.pid }::int)` +// await delay(50) +// +// const x2 = await sql`select 1` +// result.push((await sql`select system_identifier as x from pg_control_system()`)[0].x) +// await s2`select pg_terminate_backend(${ x2.state.pid }::int)` +// await delay(50) +// +// result.push((await sql`select system_identifier as x from pg_control_system()`)[0].x) +// +// return [[id1, id2, id1].join(','), result.join(',')] +//}) + +t('Escaping supports schemas and tables', async() => { + await sql`create schema a` + await sql`create table a.b (c int)` + await sql`insert into a.b (c) values (1)` + return [ + 1, + (await sql`select ${ sql('a.b.c') } from a.b`)[0].c, + await sql`drop table a.b`, + await sql`drop schema a` + ] +}) + +t('Raw method returns rows as arrays', async() => { + const [x] = await sql`select 1`.raw() + return [ + Array.isArray(x), + true + ] +}) + +t('Raw method returns values unparsed as Buffer', async() => { + const [[x]] = await sql`select 1`.raw() + return [ + x instanceof Uint8Array, + true + ] +}) + +t('Array returns rows as arrays of columns', async() => { + return [(await sql`select 1`.values())[0][0], 1] +}) + +t('Copy read', async() => { + const result = [] + + await sql`create table test (x int)` + await sql`insert into test select * from generate_series(1,10)` + const readable = await sql`copy test to stdout`.readable() + readable.on('data', x => result.push(x)) + await new Promise(r => readable.on('end', r)) + + return [ + result.length, + 10, + await sql`drop table test` + ] +}) + +t('Copy write', { timeout: 2 }, async() => { + await sql`create table test (x int)` + const writable = await sql`copy test from stdin`.writable() + + writable.write('1\n') + writable.write('1\n') + writable.end() + + await new Promise(r => writable.on('finish', r)) + + return [ + (await sql`select 1 from test`).length, + 2, + await sql`drop table test` + ] +}) + +t('Copy write as first', async() => { + await sql`create table test (x int)` + const first = postgres(options) + const writable = await first`COPY test FROM STDIN WITH(FORMAT csv, HEADER false, DELIMITER ',')`.writable() + writable.write('1\n') + writable.write('1\n') + writable.end() + + await new Promise(r => writable.on('finish', r)) + + return [ + (await sql`select 1 from test`).length, + 2, + await sql`drop table test` + ] +}) + +t('Copy from file', async() => { + await sql`create table test (x int, y int, z int)` + await new Promise(async r => fs + .createReadStream(rel('copy.csv')) + .pipe(await sql`copy test from stdin`.writable()) + .on('finish', r) + ) + + return [ + JSON.stringify(await sql`select * from test`), + '[{"x":1,"y":2,"z":3},{"x":4,"y":5,"z":6}]', + await sql`drop table test` + ] +}) + +t('Copy from works in transaction', async() => { + await sql`create table test(x int)` + const xs = await sql.begin(async sql => { + (await sql`copy test from stdin`.writable()).end('1\n2') + await delay(20) + return sql`select 1 from test` + }) + + return [ + xs.length, + 2, + await sql`drop table test` + ] +}) + +t('Copy from abort', async() => { + const sql = postgres(options) + const readable = fs.createReadStream(rel('copy.csv')) + + await sql`create table test (x int, y int, z int)` + await sql`TRUNCATE TABLE test` + + const writable = await sql`COPY test FROM STDIN`.writable() + + let aborted + + readable + .pipe(writable) + .on('error', (err) => aborted = err) + + writable.destroy(new Error('abort')) + await sql.end() + + return [ + 'abort', + aborted.message, + await postgres(options)`drop table test` + ] +}) + +t('multiple queries before connect', async() => { + const sql = postgres({ ...options, max: 2 }) + const xs = await Promise.all([ + sql`select 1 as x`, + sql`select 2 as x`, + sql`select 3 as x`, + sql`select 4 as x` + ]) + + return [ + '1,2,3,4', + xs.map(x => x[0].x).join() + ] +}) + +// Reason: No subscriptions, use Supabase Realtime +//t('subscribe', { timeout: 2 }, async() => { +// const sql = postgres({ +// database: 'postgres', +// publications: 'alltables' +// }) +// +// await sql.unsafe('create publication alltables for all tables') +// +// const result = [] +// +// const { unsubscribe } = await sql.subscribe('*', (row, { command, old }) => { +// result.push(command, row.name, row.id, old && old.name, old && old.id) +// }) +// +// await sql` +// create table test ( +// id serial primary key, +// name text +// ) +// ` +// +// await sql`alter table test replica identity default` +// await sql`insert into test (name) values ('Murray')` +// await sql`update test set name = 'Rothbard'` +// await sql`update test set id = 2` +// await sql`delete from test` +// await sql`alter table test replica identity full` +// await sql`insert into test (name) values ('Murray')` +// await sql`update test set name = 'Rothbard'` +// await sql`delete from test` +// await delay(10) +// await unsubscribe() +// await sql`insert into test (name) values ('Oh noes')` +// await delay(10) +// return [ +// 'insert,Murray,1,,,update,Rothbard,1,,,update,Rothbard,2,,1,delete,,2,,,insert,Murray,2,,,update,Rothbard,2,Murray,2,delete,Rothbard,2,,', // eslint-disable-line +// result.join(','), +// await sql`drop table test`, +// await sql`drop publication alltables`, +// await sql.end() +// ] +//}) +// +//t('subscribe with transform', { timeout: 2 }, async() => { +// const sql = postgres({ +// transform: { +// column: { +// from: postgres.toCamel, +// to: postgres.fromCamel +// } +// }, +// database: 'postgres', +// publications: 'alltables' +// }) +// +// await sql.unsafe('create publication alltables for all tables') +// +// const result = [] +// +// const { unsubscribe } = await sql.subscribe('*', (row, { command, old }) => +// result.push(command, row.nameInCamel || row.id, old && old.nameInCamel) +// ) +// +// await sql` +// create table test ( +// id serial primary key, +// name_in_camel text +// ) +// ` +// +// await sql`insert into test (name_in_camel) values ('Murray')` +// await sql`update test set name_in_camel = 'Rothbard'` +// await sql`delete from test` +// await sql`alter table test replica identity full` +// await sql`insert into test (name_in_camel) values ('Murray')` +// await sql`update test set name_in_camel = 'Rothbard'` +// await sql`delete from test` +// await delay(10) +// await unsubscribe() +// await sql`insert into test (name_in_camel) values ('Oh noes')` +// await delay(10) +// return [ +// 'insert,Murray,,update,Rothbard,,delete,1,,insert,Murray,,update,Rothbard,Murray,delete,Rothbard,', +// result.join(','), +// await sql`drop table test`, +// await sql`drop publication alltables`, +// await sql.end() +// ] +//}) +// +//t('subscribe reconnects and calls onsubscribe', { timeout: 4 }, async() => { +// const sql = postgres({ +// database: 'postgres', +// publications: 'alltables', +// fetch_types: false +// }) +// +// await sql.unsafe('create publication alltables for all tables') +// +// const result = [] +// let onsubscribes = 0 +// +// const { unsubscribe, sql: subscribeSql } = await sql.subscribe( +// '*', +// (row, { command, old }) => result.push(command, row.name || row.id, old && old.name), +// () => onsubscribes++ +// ) +// +// await sql` +// create table test ( +// id serial primary key, +// name text +// ) +// ` +// +// await sql`insert into test (name) values ('Murray')` +// await delay(10) +// await subscribeSql.close() +// await delay(500) +// await sql`delete from test` +// await delay(100) +// await unsubscribe() +// return [ +// '2insert,Murray,,delete,1,', +// onsubscribes + result.join(','), +// await sql`drop table test`, +// await sql`drop publication alltables`, +// await sql.end() +// ] +//}) + +t('Execute', async() => { + const result = await new Promise((resolve) => { + const sql = postgres({ ...options, fetch_types: false, debug:(id, query) => resolve(query) }) + sql`select 1`.execute() + }) + + return [result, 'select 1'] +}) + +t('Cancel running query', async() => { + const query = sql`select pg_sleep(2)` + setTimeout(() => query.cancel(), 500) + const error = await query.catch(x => x) + return ['57014', error.code] +}) + +t('Cancel piped query', { timeout: 5 }, async() => { + await sql`select 1` + const last = sql`select pg_sleep(1)`.execute() + const query = sql`select pg_sleep(2) as dig` + setTimeout(() => query.cancel(), 500) + const error = await query.catch(x => x) + await last + return ['57014', error.code] +}) + +t('Cancel queued query', async() => { + const query = sql`select pg_sleep(2) as nej` + const tx = sql.begin(sql => ( + query.cancel(), + sql`select pg_sleep(0.5) as hej, 'hejsa'` + )) + const error = await query.catch(x => x) + await tx + return ['57014', error.code] +}) + +t('Fragments', async() => [ + 1, + (await sql` + ${ sql`select` } 1 as x + `)[0].x +]) + +t('Result becomes array', async() => [ + true, + (await sql`select 1`).slice() instanceof Array +]) + +t('Describe', async() => { + const type = (await sql`select ${ 1 }::int as x`.describe()).types[0] + return [23, type] +}) + +t('Describe a statement', async() => { + await sql`create table tester (name text, age int)` + const r = await sql`select name, age from tester where name like $1 and age > $2`.describe() + return [ + '25,23/name:25,age:23', + `${ r.types.join(',') }/${ r.columns.map(c => `${c.name}:${c.type}`).join(',') }`, + await sql`drop table tester` + ] +}) + +t('Include table oid and column number in column details', async() => { + await sql`create table tester (name text, age int)` + const r = await sql`select name, age from tester where name like $1 and age > $2`.describe() + const [{ oid }] = await sql`select oid from pg_class where relname = 'tester'` + + return [ + `table:${oid},number:1|table:${oid},number:2`, + `${ r.columns.map(c => `table:${c.table},number:${c.number}`).join('|') }`, + await sql`drop table tester` + ] +}) + +t('Describe a statement without parameters', async() => { + await sql`create table tester (name text, age int)` + const r = await sql`select name, age from tester`.describe() + return [ + '0,2', + `${ r.types.length },${ r.columns.length }`, + await sql`drop table tester` + ] +}) + +t('Describe a statement without columns', async() => { + await sql`create table tester (name text, age int)` + const r = await sql`insert into tester (name, age) values ($1, $2)`.describe() + return [ + '2,0', + `${ r.types.length },${ r.columns.length }`, + await sql`drop table tester` + ] +}) + +t('Large object', async() => { + const file = rel('index.js') + , md5 = crypto.createHash('md5').update(fs.readFileSync(file)).digest('hex') + + const lo = await sql.largeObject() + await new Promise(async r => fs.createReadStream(file).pipe(await lo.writable()).on('finish', r)) + await lo.seek(0) + + const out = crypto.createHash('md5') + await new Promise(r => lo.readable().then(x => x.on('data', x => out.update(x)).on('end', r))) + + return [ + md5, + out.digest('hex'), + await lo.close() + ] +}) + +t('Catches type serialize errors', async() => { + const sql = postgres({ + ...options, + idle_timeout, + types: { + text: { + from: 25, + to: 25, + parse: x => x, + serialize: () => { throw new Error('watSerialize') } + } + } + }) + + return [ + 'watSerialize', + (await sql`select ${ 'wat' }`.catch(e => e.message)) + ] +}) + +t('Catches type parse errors', async() => { + const sql = postgres({ + ...options, + idle_timeout, + types: { + text: { + from: 25, + to: 25, + parse: () => { throw new Error('watParse') }, + serialize: x => x + } + } + }) + + return [ + 'watParse', + (await sql`select 'wat'`.catch(e => e.message)) + ] +}) + +t('Catches type serialize errors in transactions', async() => { + const sql = postgres({ + ...options, + idle_timeout, + types: { + text: { + from: 25, + to: 25, + parse: x => x, + serialize: () => { throw new Error('watSerialize') } + } + } + }) + + return [ + 'watSerialize', + (await sql.begin(sql => ( + sql`select 1`, + sql`select ${ 'wat' }` + )).catch(e => e.message)) + ] +}) + +t('Catches type parse errors in transactions', async() => { + const sql = postgres({ + ...options, + idle_timeout, + types: { + text: { + from: 25, + to: 25, + parse: () => { throw new Error('watParse') }, + serialize: x => x + } + } + }) + + return [ + 'watParse', + (await sql.begin(sql => ( + sql`select 1`, + sql`select 'wat'` + )).catch(e => e.message)) + ] +}) + +t('Prevent premature end of connection in transaction', async() => { + const sql = postgres({...options, max_lifetime: 0.01, idle_timeout }) + const result = await sql.begin(async sql => { + await sql`select 1` + await delay(20) + await sql`select 1` + return 'yay' + }) + + + return [ + 'yay', + result + ] +}) + +t('Ensure reconnect after max_lifetime with transactions', { timeout: 5 }, async() => { + const sql = postgres({ + ...options, + max_lifetime: 0.01, + idle_timeout, + max: 1 + }) + + let x = 0 + while (x++ < 10) await sql.begin(sql => sql`select 1 as x`) + + return [true, true] +}) + + +t('Ensure transactions throw if connection is closed dwhile there is no query', async() => { + const sql = postgres(options) + const x = await sql.begin(async() => { + setTimeout(() => sql.end({ timeout: 0 }), 10) + await new Promise(r => setTimeout(r, 200)) + return sql`select 1` + }).catch(x => x) + return ['CONNECTION_CLOSED', x.code] +}) + +t('Custom socket', {}, async() => { + let result + const sql = postgres({ + ...options, + socket: () => new Promise((resolve, reject) => { + const socket = new net.Socket() + socket.connect(options.port) + socket.once('data', x => result = x[0]) + socket.on('error', reject) + socket.on('connect', () => resolve(socket)) + }), + idle_timeout + }) + + await sql`select 1` + + return [ + result, + 82 + ] +}) + +t('Ensure drain only dequeues if ready', async() => { + const sql = postgres(options) + + const res = await Promise.all([ + sql.unsafe('SELECT 0+$1 --' + '.'.repeat(100000), [1]), + sql.unsafe('SELECT 0+$1+$2+$3', [1, 2, 3]) + ]) + + return [res.length, 2] +}) + +t('Supports fragments as dynamic parameters', async() => { + await sql`create table test (a int, b bool)` + await sql`insert into test values(1, true)` + await sql`insert into test ${ + sql({ + a: 2, + b: sql`exists(select 1 from test where b = ${ true })` + }) + }` + + return [ + '1,t2,t', + (await sql`select * from test`.raw()).join(''), + await sql`drop table test` + ] +}) + +t('Supports nested fragments with parameters', async() => { + await sql`create table test ${ + sql`(${ sql('a') } ${ sql`int` })` + }` + await sql`insert into test values(1)` + return [ + 1, + (await sql`select a from test`)[0].a, + await sql`drop table test` + ] +}) + +t('Supports multiple nested fragments with parameters', async() => { + const [{ b }] = await sql`select * ${ + sql`from ${ + sql`(values (2, ${ 1 }::int)) as x(${ sql(['a', 'b']) })` + }` + }` + return [ + 1, + b + ] +}) + +t('Supports arrays of fragments', async() => { + const [{ x }] = await sql` + ${ [sql`select`, sql`1`, sql`as`, sql`x`] } + ` + + return [ + 1, + x + ] +}) + +t('Does not try rollback when commit errors', async() => { + let notice = null + const sql = postgres({ ...options, onnotice: x => notice = x }) + await sql`create table test(x int constraint test_constraint unique deferrable initially deferred)` + + await sql.begin('isolation level serializable', async sql => { + await sql`insert into test values(1)` + await sql`insert into test values(1)` + }).catch(e => e) + + return [ + notice, + null, + await sql`drop table test` + ] +}) + +t('Last keyword used even with duplicate keywords', async() => { + await sql`create table test (x int)` + await sql`insert into test values(1)` + const [{ x }] = await sql` + select + 1 in (1) as x + from test + where x in ${ sql([1, 2]) } + ` + + return [x, true, await sql`drop table test`] +}) + +t('Insert array with null', async() => { + await sql`create table test (x int[])` + await sql`insert into test ${ sql({ x: [1, null, 3] }) }` + return [ + 1, + (await sql`select x from test`)[0].x[0], + await sql`drop table test` + ] +}) + +t('Insert array with undefined throws', async() => { + await sql`create table test (x int[])` + return [ + 'UNDEFINED_VALUE', + await sql`insert into test ${ sql({ x: [1, undefined, 3] }) }`.catch(e => e.code), + await sql`drop table test` + ] +}) + +t('Insert array with undefined transform', async() => { + const sql = postgres({ ...options, transform: { undefined: null } }) + await sql`create table test (x int[])` + await sql`insert into test ${ sql({ x: [1, undefined, 3] }) }` + return [ + 1, + (await sql`select x from test`)[0].x[0], + await sql`drop table test` + ] +}) + +t('concurrent cursors', async() => { + const xs = [] + + await Promise.all([...Array(7)].map((x, i) => [ + sql`select ${ i }::int as a, generate_series(1, 2) as x`.cursor(([x]) => xs.push(x.a + x.x)) + ]).flat()) + + return ['12233445566778', xs.join('')] +}) + +t('concurrent cursors multiple connections', async() => { + const sql = postgres({ ...options, max: 2 }) + const xs = [] + + await Promise.all([...Array(7)].map((x, i) => [ + sql`select ${ i }::int as a, generate_series(1, 2) as x`.cursor(([x]) => xs.push(x.a + x.x)) + ]).flat()) + + return ['12233445566778', xs.sort().join('')] +}) + +t('reserve connection', async() => { + const reserved = await sql.reserve() + + setTimeout(() => reserved.release(), 510) + + const xs = await Promise.all([ + reserved`select 1 as x`.then(([{ x }]) => ({ time: Date.now(), x })), + sql`select 2 as x`.then(([{ x }]) => ({ time: Date.now(), x })), + reserved`select 3 as x`.then(([{ x }]) => ({ time: Date.now(), x })) + ]) + + if (xs[1].time - xs[2].time < 500) + throw new Error('Wrong time') + + return [ + '123', + xs.map(x => x.x).join('') + ] +}) + +t('arrays in reserved connection', async() => { + const reserved = await sql.reserve() + const [{ x }] = await reserved`select array[1, 2, 3] as x` + reserved.release() + + return [ + '123', + x.join('') + ] +}) diff --git a/test/integration/js/postgres/select-param.sql b/test/integration/js/postgres/select-param.sql new file mode 100644 index 00000000..d4de2440 --- /dev/null +++ b/test/integration/js/postgres/select-param.sql @@ -0,0 +1 @@ +select $1 as x diff --git a/test/integration/js/postgres/select.sql b/test/integration/js/postgres/select.sql new file mode 100644 index 00000000..f951e920 --- /dev/null +++ b/test/integration/js/postgres/select.sql @@ -0,0 +1 @@ +select 1 as x diff --git a/test/integration/js/postgres/test.js b/test/integration/js/postgres/test.js new file mode 100644 index 00000000..9accbe86 --- /dev/null +++ b/test/integration/js/postgres/test.js @@ -0,0 +1,87 @@ +/* eslint no-console: 0 */ + +import util from 'node:util' + +let done = 0 +let only = false +let ignored = 0 +let failed = false +let promise = Promise.resolve() +const tests = {} + , ignore = {} + +export const nt = () => ignored++ +export const ot = (...rest) => (only = true, test(true, ...rest)) +export const t = (...rest) => test(false, ...rest) +t.timeout = 5 + +async function test(o, name, options, fn) { + typeof options !== 'object' && (fn = options, options = {}) + const line = new Error().stack.split('\n')[3].match(':([0-9]+):')[1] + + await 1 + + if (only && !o) + return + + tests[line] = { fn, line, name } + promise = promise.then(() => Promise.race([ + new Promise((resolve, reject) => + fn.timer = setTimeout(() => reject('Timed out'), (options.timeout || t.timeout) * 1000) + ), + failed + ? (ignored++, ignore) + : fn() + ])) + .then(async x => { + clearTimeout(fn.timer) + if (x === ignore) + return + + if (!Array.isArray(x)) + throw new Error('Test should return result array') + + const [expected, got] = await Promise.all(x) + if (expected !== got) { + failed = true + throw new Error(util.inspect(expected) + ' != ' + util.inspect(got)) + } + + tests[line].succeeded = true + process.stdout.write('✅') + }) + .catch(err => { + tests[line].failed = failed = true + tests[line].error = err instanceof Error ? err : new Error(util.inspect(err)) + }) + .then(() => { + ++done === Object.keys(tests).length && exit() + }) +} + +function exit() { + let success = true + Object.values(tests).every((x) => { + if (x.succeeded) + return true + + success = false + x.cleanup + ? console.error('⛔️', x.name + ' at line', x.line, 'cleanup failed', '\n', util.inspect(x.cleanup)) + : console.error('⛔️', x.name + ' at line', x.line, x.failed + ? 'failed' + : 'never finished', x.error ? '\n' + util.inspect(x.error) : '' + ) + }) + + only + ? console.error('⚠️', 'Not all tests were run') + : ignored + ? console.error('⚠️', ignored, 'ignored test' + (ignored === 1 ? '' : 's'), '\n') + : success + ? console.log('🎉') + : console.error('⚠️', 'Not good') + + !process.exitCode && (!success || only || ignored) && (process.exitCode = 1) +} + diff --git a/test/integration/js/yarn.lock b/test/integration/js/yarn.lock new file mode 100644 index 00000000..187e2cfc --- /dev/null +++ b/test/integration/js/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +postgres@^3.4.5: + version "3.4.5" + resolved "https://registry.yarnpkg.com/postgres/-/postgres-3.4.5.tgz#1ef99e51b0ba9b53cbda8a215dd406725f7d15f9" + integrity sha512-cDWgoah1Gez9rN3H4165peY9qfpEo+SA61oQv65O3cRUE1pOEoJWwddwcqKE8XZYjbblOJlYDlLV4h67HrEVDg== diff --git a/test/test_helper.exs b/test/test_helper.exs index d7449a11..b4467cac 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -4,7 +4,10 @@ Cachex.start_link(name: Supavisor.Cache) ExUnit.start( capture_log: true, - exclude: [flaky: true] + exclude: [ + flaky: true, + integration: true + ] ) Ecto.Adapters.SQL.Sandbox.mode(Supavisor.Repo, :auto) diff --git a/typos.toml b/typos.toml index 6b76359c..ec32404b 100644 --- a/typos.toml +++ b/typos.toml @@ -1,3 +1,9 @@ +[files] +extend-exclude = [ + # Ignore integration tests, as these can be copied as-is + "test/integration/**" +] + [default] extend-ignore-re = [ "\\bey[A-Za-z0-9_-]{20,}\\.ey[A-Za-z0-9_-]{20,}\\.[A-Za-z0-9_-]{20,}\\b", From 3729f9f68718849dc695678eb34eecd8a1e71506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Mon, 18 Nov 2024 13:32:37 +0100 Subject: [PATCH 92/97] chore: release v2.0.6 (#485) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e0102586..157e54f3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.5 +2.0.6 From 9252c05c29b146306c1597ccd04a8d912608df42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 19 Nov 2024 17:29:46 +0100 Subject: [PATCH 93/97] fix: shutdown Postgrex connection on failure (#487) Handle errors during authentication query gracefully and shutdown the client process to not leave the hanging connections. --- lib/supavisor/client_handler.ex | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 6ab3ad4e..f4864395 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -957,22 +957,26 @@ defmodule Supavisor.ClientHandler do ssl_opts: ssl_opts || [] ) - # kill the postgrex connection if the current process exits unexpectedly - Process.link(conn) - - Logger.debug( - "ClientHandler: Connected to db #{tenant.db_host} #{tenant.db_port} #{tenant.db_database} #{user.db_user}" - ) + try do + Logger.debug( + "ClientHandler: Connected to db #{tenant.db_host} #{tenant.db_port} #{tenant.db_database} #{user.db_user}" + ) - resp = - with {:ok, secret} <- Helpers.get_user_secret(conn, tenant.auth_query, db_user) do - t = if secret.digest == :md5, do: :auth_query_md5, else: :auth_query - {:ok, {t, fn -> Map.put(secret, :alias, user.db_user_alias) end}} - end + resp = + with {:ok, secret} <- Helpers.get_user_secret(conn, tenant.auth_query, db_user) do + t = if secret.digest == :md5, do: :auth_query_md5, else: :auth_query + {:ok, {t, fn -> Map.put(secret, :alias, user.db_user_alias) end}} + end - GenServer.stop(conn, :normal, 5_000) - Logger.info("ClientHandler: Get secrets finished") - resp + Logger.info("ClientHandler: Get secrets finished") + resp + rescue + exception -> + Logger.error("ClientHandler: Couldn't fetch user secrets from #{tenant.db_host}") + reraise exception, __STACKTRACE__ + after + GenServer.stop(conn, :normal, 5_000) + end end @spec exchange_first(:password | :auth_query, fun(), binary(), binary(), binary()) :: From bf89e0c10180636aa800276c0e289e459c1442a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 19 Nov 2024 17:31:54 +0100 Subject: [PATCH 94/97] chore: release v2.0.7 (#488) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 157e54f3..f1547e6d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.6 +2.0.7 From 72273f3b86da1eadbba80943b0d1d616ace89825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Thu, 21 Nov 2024 12:50:16 +0100 Subject: [PATCH 95/97] test: update Postgres.js tests (#491) * test: increase pool size in external test suite to match nano instances * test: make timeouts configurable * test: disable prepared transactions test, as these are disabled for security reasons * test: disable cursor tests in transaction mode * test: disable custom socket test * test: disable MAX_PARAMETERS_EXCEEDED test * test: disable query format errors test * test: run connection reserving test only in session mode * test: make Postgres.js more verbose * test: ensure connections with different parameters are available where needed * test: fix typo in test name --- test/integration/external_test.exs | 2 +- test/integration/js/postgres/index.js | 377 +++++++++++++------------- test/integration/js/postgres/test.js | 23 +- 3 files changed, 206 insertions(+), 196 deletions(-) diff --git a/test/integration/external_test.exs b/test/integration/external_test.exs index 30299bc1..5601acca 100644 --- a/test/integration/external_test.exs +++ b/test/integration/external_test.exs @@ -111,7 +111,7 @@ defmodule Supavisor.Integration.ExternalTest do external_id: external_id, users: [ %{ - "pool_size" => 3, + "pool_size" => 15, "db_user" => "postgres", "db_password" => "postgres", "is_manager" => true, diff --git a/test/integration/js/postgres/index.js b/test/integration/js/postgres/index.js index a8f68216..4d56013c 100644 --- a/test/integration/js/postgres/index.js +++ b/test/integration/js/postgres/index.js @@ -7,7 +7,7 @@ import postgres from 'postgres' const delay = ms => new Promise(r => setTimeout(r, ms)) const rel = x => new URL(x, import.meta.url) -const idle_timeout = 1 +const idle_timeout = t.timeout const login = { user: process.env.PGUSER, @@ -32,7 +32,7 @@ const options = { user: login.user, pass: login.pass, idle_timeout, - connect_timeout: 1, + connect_timeout: t.timeout, max: 1 } @@ -75,7 +75,7 @@ t('Create table', async() => ['CREATE TABLE', (await sql`create table test(int int)`).command, await sql`drop table test`] ) -t('Drop table', { timeout: 2 }, async() => { +t('Drop table', { timeout: t.timeout * 2 }, async() => { await sql`create table test(int int)` return ['DROP TABLE', (await sql`drop table test`).command] }) @@ -242,20 +242,20 @@ t('Savepoint returns Result', async() => { return [1, result[0].x] }) -if (options.prepare) { - t('Prepared transaction', async() => { - await sql`create table test (a int)` - - await sql.begin(async sql => { - await sql`insert into test values(1)` - await sql.prepare('tx1') - }) - - await sql`commit prepared 'tx1'` - - return ['1', (await sql`select count(1) from test`)[0].count, await sql`drop table test`] - }) -} +// Reason: disabled because of security reasons +// +//t('Prepared transaction', async() => { +// await sql`create table test (a int)` +// +// await sql.begin(async sql => { +// await sql`insert into test values(1)` +// await sql.prepare('tx1') +// }) +// +// await sql`commit prepared 'tx1'` +// +// return ['1', (await sql`select count(1) from test`)[0].count, await sql`drop table test`] +//}) t('Transaction requests are executed implicitly', async() => { const sql = postgres({ ...options, debug: true, idle_timeout: 1, fetch_types: false }) @@ -941,7 +941,7 @@ if (options.prepare) { // return ['postgres.js', (await sql`select 1`.then(() => sql.parameters.application_name))] //}) -t('big query body', { timeout: 2 }, async() => { +t('big query body', { timeout: t.timeout * 2 }, async() => { const size = 50000 await sql`create table test (x int)` return [size, (await sql`insert into test ${ @@ -949,12 +949,13 @@ t('big query body', { timeout: 2 }, async() => { }`).count, await sql`drop table test`] }) -t('Throws if more than 65534 parameters', async() => { - await sql`create table test (x int)` - return ['MAX_PARAMETERS_EXCEEDED', (await sql`insert into test ${ - sql([...Array(65535).keys()].map(x => ({ x }))) - }`.catch(e => e.code)), await sql`drop table test`] -}) +// Reason: This tests checks internalt of the library, not the DB stuff +//ot('Throws if more than 65534 parameters', {timeout: t.timeout * 2}, async() => { +// await sql`create table test (x int) -- barfoo ` +// return ['MAX_PARAMETERS_EXCEEDED', (await sql`insert into test ${ +// sql([...Array(65535).keys()].map(x => ({ x }))) +// }`.catch(e => (console.debug(e.code), e.code)))] //, await sql`drop table test -- foobar`] +//}) t('let postgres do implicit cast of unknown types', async() => { await sql`create table test (x timestamp with time zone)` @@ -1019,7 +1020,7 @@ t('little bobby tables', async() => { }) t('Connection errors are caught using begin()', { - timeout: 2 + timeout: t.timeout * 2 }, async() => { let error try { @@ -1218,6 +1219,7 @@ t('dynamic values multi row', async() => { //}) t('Multiple queries', async() => { + const sql = postgres({ ...options, max: 5 }) return [4, (await Promise.all([ sql`select 1`, sql`select 2`, @@ -1293,145 +1295,147 @@ t('forEach returns empty array', async() => { return [0, (await sql`select 1 as x`.forEach(() => { /* noop */ })).length] }) -t('Cursor', async() => { - const order = [] - await sql`select 1 as x union select 2 as x`.cursor(async([x]) => { - order.push(x.x + 'a') - await delay(100) - order.push(x.x + 'b') +if (process.env.PGMODE != 'transaction') { + t('Cursor', async() => { + const order = [] + await sql`select 1 as x union select 2 as x`.cursor(async([x]) => { + order.push(x.x + 'a') + await delay(100) + order.push(x.x + 'b') + }) + return ['1a1b2a2b', order.join('')] }) - return ['1a1b2a2b', order.join('')] -}) -t('Unsafe cursor', async() => { - const order = [] - await sql.unsafe('select 1 as x union select 2 as x').cursor(async([x]) => { - order.push(x.x + 'a') - await delay(100) - order.push(x.x + 'b') + t('Unsafe cursor', async() => { + const order = [] + await sql.unsafe('select 1 as x union select 2 as x').cursor(async([x]) => { + order.push(x.x + 'a') + await delay(100) + order.push(x.x + 'b') + }) + return ['1a1b2a2b', order.join('')] }) - return ['1a1b2a2b', order.join('')] -}) -t('Cursor custom n', async() => { - const order = [] - await sql`select * from generate_series(1,20)`.cursor(10, async(x) => { - order.push(x.length) + t('Cursor custom n', async() => { + const order = [] + await sql`select * from generate_series(1,20)`.cursor(10, async(x) => { + order.push(x.length) + }) + return ['10,10', order.join(',')] }) - return ['10,10', order.join(',')] -}) -t('Cursor custom with rest n', async() => { - const order = [] - await sql`select * from generate_series(1,20)`.cursor(11, async(x) => { - order.push(x.length) + t('Cursor custom with rest n', async() => { + const order = [] + await sql`select * from generate_series(1,20)`.cursor(11, async(x) => { + order.push(x.length) + }) + return ['11,9', order.join(',')] }) - return ['11,9', order.join(',')] -}) -t('Cursor custom with less results than batch size', async() => { - const order = [] - await sql`select * from generate_series(1,20)`.cursor(21, async(x) => { - order.push(x.length) + t('Cursor custom with less results than batch size', async() => { + const order = [] + await sql`select * from generate_series(1,20)`.cursor(21, async(x) => { + order.push(x.length) + }) + return ['20', order.join(',')] }) - return ['20', order.join(',')] -}) -t('Cursor cancel', async() => { - let result - await sql`select * from generate_series(1,10) as x`.cursor(async([{ x }]) => { - result = x - return sql.CLOSE + t('Cursor cancel', async() => { + let result + await sql`select * from generate_series(1,10) as x`.cursor(async([{ x }]) => { + result = x + return sql.CLOSE + }) + return [1, result] }) - return [1, result] -}) -t('Cursor throw', async() => { - const order = [] - await sql`select 1 as x union select 2 as x`.cursor(async([x]) => { - order.push(x.x + 'a') - await delay(100) - throw new Error('watty') - }).catch(() => order.push('err')) - return ['1aerr', order.join('')] -}) - -t('Cursor error', async() => [ - '42601', - await sql`wat`.cursor(() => { /* noop */ }).catch((err) => err.code) -]) + t('Cursor throw', async() => { + const order = [] + await sql`select 1 as x union select 2 as x`.cursor(async([x]) => { + order.push(x.x + 'a') + await delay(100) + throw new Error('watty') + }).catch(() => order.push('err')) + return ['1aerr', order.join('')] + }) -t('Multiple Cursors', { timeout: 2 }, async() => { - const result = [] - await sql.begin(async sql => [ - await sql`select 1 as cursor, x from generate_series(1,4) as x`.cursor(async([row]) => { - result.push(row.x) - await new Promise(r => setTimeout(r, 20)) - }), - await sql`select 2 as cursor, x from generate_series(101,104) as x`.cursor(async([row]) => { - result.push(row.x) - await new Promise(r => setTimeout(r, 10)) - }) + t('Cursor error', async() => [ + '42601', + await sql`wat`.cursor(() => { /* noop */ }).catch((err) => err.code) ]) - return ['1,2,3,4,101,102,103,104', result.join(',')] -}) + t('Multiple Cursors', { timeout: t.timeout * 2 }, async() => { + const result = [] + await sql.begin(async sql => [ + await sql`select 1 as cursor, x from generate_series(1,4) as x`.cursor(async([row]) => { + result.push(row.x) + await new Promise(r => setTimeout(r, 20)) + }), + await sql`select 2 as cursor, x from generate_series(101,104) as x`.cursor(async([row]) => { + result.push(row.x) + await new Promise(r => setTimeout(r, 10)) + }) + ]) + + return ['1,2,3,4,101,102,103,104', result.join(',')] + }) -t('Cursor as async iterator', async() => { - const order = [] - for await (const [x] of sql`select generate_series(1,2) as x;`.cursor()) { - order.push(x.x + 'a') - await delay(10) - order.push(x.x + 'b') - } + t('Cursor as async iterator', async() => { + const order = [] + for await (const [x] of sql`select generate_series(1,2) as x;`.cursor()) { + order.push(x.x + 'a') + await delay(10) + order.push(x.x + 'b') + } - return ['1a1b2a2b', order.join('')] -}) + return ['1a1b2a2b', order.join('')] + }) -t('Cursor as async iterator with break', async() => { - const order = [] - for await (const xs of sql`select generate_series(1,2) as x;`.cursor()) { - order.push(xs[0].x + 'a') - await delay(10) - order.push(xs[0].x + 'b') - break - } + t('Cursor as async iterator with break', async() => { + const order = [] + for await (const xs of sql`select generate_series(1,2) as x;`.cursor()) { + order.push(xs[0].x + 'a') + await delay(10) + order.push(xs[0].x + 'b') + break + } - return ['1a1b', order.join('')] -}) + return ['1a1b', order.join('')] + }) -t('Async Iterator Unsafe cursor', async() => { - const order = [] - for await (const [x] of sql.unsafe('select 1 as x union select 2 as x').cursor()) { - order.push(x.x + 'a') - await delay(10) - order.push(x.x + 'b') - } - return ['1a1b2a2b', order.join('')] -}) + t('Async Iterator Unsafe cursor', async() => { + const order = [] + for await (const [x] of sql.unsafe('select 1 as x union select 2 as x').cursor()) { + order.push(x.x + 'a') + await delay(10) + order.push(x.x + 'b') + } + return ['1a1b2a2b', order.join('')] + }) -t('Async Iterator Cursor custom n', async() => { - const order = [] - for await (const x of sql`select * from generate_series(1,20)`.cursor(10)) + t('Async Iterator Cursor custom n', async() => { + const order = [] + for await (const x of sql`select * from generate_series(1,20)`.cursor(10)) order.push(x.length) - return ['10,10', order.join(',')] -}) + return ['10,10', order.join(',')] + }) -t('Async Iterator Cursor custom with rest n', async() => { - const order = [] - for await (const x of sql`select * from generate_series(1,20)`.cursor(11)) + t('Async Iterator Cursor custom with rest n', async() => { + const order = [] + for await (const x of sql`select * from generate_series(1,20)`.cursor(11)) order.push(x.length) - return ['11,9', order.join(',')] -}) + return ['11,9', order.join(',')] + }) -t('Async Iterator Cursor custom with less results than batch size', async() => { - const order = [] - for await (const x of sql`select * from generate_series(1,20)`.cursor(21)) + t('Async Iterator Cursor custom with less results than batch size', async() => { + const order = [] + for await (const x of sql`select * from generate_series(1,20)`.cursor(21)) order.push(x.length) - return ['20', order.join(',')] -}) + return ['20', order.join(',')] + }) +} t('Transform row', async() => { const sql = postgres({ @@ -1627,7 +1631,7 @@ t('Query and parameters are enumerable if debug is set', async() => { ] }) -t('connect_timeout', { timeout: 20 }, async() => { +t('connect_timeout', { timeout: t.timeout * 20 }, async() => { const connect_timeout = 0.2 const server = net.createServer() server.listen() @@ -1651,7 +1655,7 @@ t('connect_timeout throws proper error', async() => [ })`select 1`.catch(e => e.code) ]) -t('connect_timeout error message includes host:port', { timeout: 20 }, async() => { +t('connect_timeout error message includes host:port', { timeout: t.timeout * 20 }, async() => { const connect_timeout = 0.2 const server = net.createServer() server.listen() @@ -1764,7 +1768,7 @@ if (options.prepare) { return [false, result.some(x => x.name = result.statement.name)] }) - t('Recreate prepared statements on transformAssignedExpr error', { timeout: 1 }, async() => { + t('Recreate prepared statements on transformAssignedExpr error', async() => { const insert = () => sql`insert into test (name) values (${ '1' }) returning name` await sql`create table test (name text)` await insert() @@ -1846,10 +1850,11 @@ t('Catches connection config errors with end', async() => { ] }) -t('Catches query format errors', async() => [ - 'wat', - await sql.unsafe({ toString: () => { throw new Error('wat') } }).catch((e) => e.message) -]) +// Reason: It tests internals of the library, not DB connection +//nt('Catches query format errors', async() => [ +// 'wat', +// await sql.unsafe({ toString: () => { throw new Error('wat') } }).catch((e) => e.message) +//]) // Reason: single host only //t('Multiple hosts', { @@ -1926,7 +1931,7 @@ t('Copy read', async() => { ] }) -t('Copy write', { timeout: 2 }, async() => { +t('Copy write', { timeout: t.timeout * 2 }, async() => { await sql`create table test (x int)` const writable = await sql`copy test from stdin`.writable() @@ -2178,7 +2183,8 @@ t('Cancel running query', async() => { return ['57014', error.code] }) -t('Cancel piped query', { timeout: 5 }, async() => { +t('Cancel piped query', { timeout: t.timeout * 2 }, async() => { + const sql = postgres({...options, max: 2}) await sql`select 1` const last = sql`select pg_sleep(1)`.execute() const query = sql`select pg_sleep(2) as dig` @@ -2378,7 +2384,7 @@ t('Prevent premature end of connection in transaction', async() => { ] }) -t('Ensure reconnect after max_lifetime with transactions', { timeout: 5 }, async() => { +t('Ensure reconnect after max_lifetime with transactions', { timeout: t.timeout * 5 }, async() => { const sql = postgres({ ...options, max_lifetime: 0.01, @@ -2393,7 +2399,7 @@ t('Ensure reconnect after max_lifetime with transactions', { timeout: 5 }, async }) -t('Ensure transactions throw if connection is closed dwhile there is no query', async() => { +t('Ensure transactions throw if connection is closed while there is no query', async() => { const sql = postgres(options) const x = await sql.begin(async() => { setTimeout(() => sql.end({ timeout: 0 }), 10) @@ -2403,27 +2409,30 @@ t('Ensure transactions throw if connection is closed dwhile there is no query', return ['CONNECTION_CLOSED', x.code] }) -t('Custom socket', {}, async() => { - let result - const sql = postgres({ - ...options, - socket: () => new Promise((resolve, reject) => { - const socket = new net.Socket() - socket.connect(options.port) - socket.once('data', x => result = x[0]) - socket.on('error', reject) - socket.on('connect', () => resolve(socket)) - }), - idle_timeout - }) - - await sql`select 1` - - return [ - result, - 82 - ] -}) +// Reason: Irrelevant to us, if user wants to use custom socket, it is up to +// them to make it work. +// +//t('Custom socket', {}, async() => { +// let result +// const sql = postgres({ +// ...options, +// socket: () => new Promise((resolve, reject) => { +// const socket = new net.Socket() +// socket.connect(options.port) +// socket.once('data', x => result = x[0]) +// socket.on('error', reject) +// socket.on('connect', () => resolve(socket)) +// }), +// idle_timeout +// }) +// +// await sql`select 1` +// +// return [ +// result, +// 82 +// ] +//}) t('Ensure drain only dequeues if ready', async() => { const sql = postgres(options) @@ -2569,25 +2578,27 @@ t('concurrent cursors multiple connections', async() => { return ['12233445566778', xs.sort().join('')] }) -t('reserve connection', async() => { - const reserved = await sql.reserve() +if (process.env.PGMODE != 'transaction') { + t('reserve connection', async() => { + const reserved = await sql.reserve() - setTimeout(() => reserved.release(), 510) + setTimeout(() => reserved.release(), 510) - const xs = await Promise.all([ - reserved`select 1 as x`.then(([{ x }]) => ({ time: Date.now(), x })), - sql`select 2 as x`.then(([{ x }]) => ({ time: Date.now(), x })), - reserved`select 3 as x`.then(([{ x }]) => ({ time: Date.now(), x })) - ]) + const xs = await Promise.all([ + reserved`select 1 as x`.then(([{ x }]) => ({ time: Date.now(), x })), + sql`select 2 as x`.then(([{ x }]) => ({ time: Date.now(), x })), + reserved`select 3 as x`.then(([{ x }]) => ({ time: Date.now(), x })) + ]) - if (xs[1].time - xs[2].time < 500) - throw new Error('Wrong time') + if (xs[1].time - xs[2].time < 500) + throw new Error('Wrong time') - return [ - '123', - xs.map(x => x.x).join('') - ] -}) + return [ + '123', + xs.map(x => x.x).join('') + ] + }) +} t('arrays in reserved connection', async() => { const reserved = await sql.reserve() diff --git a/test/integration/js/postgres/test.js b/test/integration/js/postgres/test.js index 9accbe86..08786ee4 100644 --- a/test/integration/js/postgres/test.js +++ b/test/integration/js/postgres/test.js @@ -8,12 +8,14 @@ let ignored = 0 let failed = false let promise = Promise.resolve() const tests = {} - , ignore = {} + , ignore = Symbol('ignore') + +const failFast = !!process.env.FAIL_FAST export const nt = () => ignored++ export const ot = (...rest) => (only = true, test(true, ...rest)) export const t = (...rest) => test(false, ...rest) -t.timeout = 5 +t.timeout = (process.env.TIMEOUT || 5) | 0 async function test(o, name, options, fn) { typeof options !== 'object' && (fn = options, options = {}) @@ -29,9 +31,10 @@ async function test(o, name, options, fn) { new Promise((resolve, reject) => fn.timer = setTimeout(() => reject('Timed out'), (options.timeout || t.timeout) * 1000) ), - failed - ? (ignored++, ignore) - : fn() + (failed && failFast) ? (ignored++, ignore) : (function() { + process.stdout.write(`${name}: `) + return fn() + })() ])) .then(async x => { clearTimeout(fn.timer) @@ -48,11 +51,13 @@ async function test(o, name, options, fn) { } tests[line].succeeded = true - process.stdout.write('✅') + process.stdout.write('✅\n') }) .catch(err => { + process.stdout.write('⛔️') tests[line].failed = failed = true tests[line].error = err instanceof Error ? err : new Error(util.inspect(err)) + console.error(name + ' at line', line, 'failed\n', util.inspect(err)) }) .then(() => { ++done === Object.keys(tests).length && exit() @@ -66,12 +71,6 @@ function exit() { return true success = false - x.cleanup - ? console.error('⛔️', x.name + ' at line', x.line, 'cleanup failed', '\n', util.inspect(x.cleanup)) - : console.error('⛔️', x.name + ' at line', x.line, x.failed - ? 'failed' - : 'never finished', x.error ? '\n' + util.inspect(x.error) : '' - ) }) only From 15802b7a34f9aafbd867ce21c66b918526907f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Fri, 22 Nov 2024 11:44:15 +0100 Subject: [PATCH 96/97] fix: use Peep for metrics gathering (#494) --- config/config.exs | 2 ++ lib/supavisor/monitoring/prom_ex.ex | 2 +- lib/supavisor/monitoring/tenant.ex | 4 ---- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 36ca234d..46291625 100644 --- a/config/config.exs +++ b/config/config.exs @@ -16,6 +16,8 @@ config :supavisor, reconnect_retries: System.get_env("RECONNECT_RETRIES", "5") |> String.to_integer(), subscribe_retries: System.get_env("SUBSCRIBE_RETRIES", "20") |> String.to_integer() +config :prom_ex, storage_adapter: PromEx.Storage.Peep + # Configures the endpoint config :supavisor, SupavisorWeb.Endpoint, url: [host: "localhost"], diff --git a/lib/supavisor/monitoring/prom_ex.ex b/lib/supavisor/monitoring/prom_ex.ex index 48b762d3..1d0faa72 100644 --- a/lib/supavisor/monitoring/prom_ex.ex +++ b/lib/supavisor/monitoring/prom_ex.ex @@ -5,7 +5,7 @@ defmodule Supavisor.Monitoring.PromEx do and provides a function to remove remote metrics associated with a specific tenant. """ - use PromEx, otp_app: :supavisor, store: PromEx.Storage.Peep + use PromEx, otp_app: :supavisor require Logger alias PromEx.Plugins diff --git a/lib/supavisor/monitoring/tenant.ex b/lib/supavisor/monitoring/tenant.ex index d4fa9544..d129b343 100644 --- a/lib/supavisor/monitoring/tenant.ex +++ b/lib/supavisor/monitoring/tenant.ex @@ -44,7 +44,6 @@ defmodule Supavisor.PromEx.Plugins.Tenant do tags: @tags, unit: {:native, :millisecond}, reporter_options: [ - buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000], peep_bucket_calculator: Buckets ] ), @@ -56,7 +55,6 @@ defmodule Supavisor.PromEx.Plugins.Tenant do tags: @tags, unit: {:native, :millisecond}, reporter_options: [ - buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000], peep_bucket_calculator: Buckets ] ), @@ -68,7 +66,6 @@ defmodule Supavisor.PromEx.Plugins.Tenant do tags: @tags, unit: {:native, :millisecond}, reporter_options: [ - buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000], peep_bucket_calculator: Buckets ] ), @@ -80,7 +77,6 @@ defmodule Supavisor.PromEx.Plugins.Tenant do tags: @tags, unit: {:native, :millisecond}, reporter_options: [ - buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000], peep_bucket_calculator: Buckets ] ), From 9453cf84b211ededfaedd3ff7c724bf0c6af9495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Mon, 25 Nov 2024 19:08:50 +0100 Subject: [PATCH 97/97] feat: change `Protocol.Server.cancel_message/2` function a macro (#495) This allows us to use this piece of code in function head in pattern match. This reduces places where the magic value is required and make the code a little bit neater and cleaner. --- lib/supavisor/client_handler.ex | 6 +++--- lib/supavisor/handler_helpers.ex | 3 ++- lib/supavisor/protocol/server.ex | 11 ++++++----- test/supavisor/protocol_test.exs | 31 +++++++++++++++++-------------- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index f4864395..35244a51 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -11,7 +11,6 @@ defmodule Supavisor.ClientHandler do @behaviour :ranch_protocol @behaviour :gen_statem @proto [:tcp, :ssl] - @cancel_query_msg <<16::32, 1234::16, 5678::16>> @switch_active_count Application.compile_env(:supavisor, :switch_active_count) @subscribe_retries Application.compile_env(:supavisor, :subscribe_retries) @timeout_subscribe 500 @@ -22,10 +21,11 @@ defmodule Supavisor.ClientHandler do Helpers, Monitoring.Telem, Protocol.Client, - Protocol.Server, Tenants } + require Supavisor.Protocol.Server, as: Server + @impl true def start_link(ref, transport, opts) do pid = :proc_lib.spawn_link(__MODULE__, :init, [ref, transport, opts]) @@ -111,7 +111,7 @@ defmodule Supavisor.ClientHandler do end # cancel request - def handle_event(:info, {_, _, <<@cancel_query_msg, pid::32, key::32>>}, _, _) do + def handle_event(:info, {_, _, Server.cancel_message(pid, key)}, _, _) do Logger.debug("ClientHandler: Got cancel query for #{inspect({pid, key})}") :ok = HandlerHelpers.send_cancel_query(pid, key) {:stop, {:shutdown, :cancel_query}} diff --git a/lib/supavisor/handler_helpers.ex b/lib/supavisor/handler_helpers.ex index 91f8055e..f46afee9 100644 --- a/lib/supavisor/handler_helpers.ex +++ b/lib/supavisor/handler_helpers.ex @@ -2,7 +2,8 @@ defmodule Supavisor.HandlerHelpers do @moduledoc false alias Phoenix.PubSub - alias Supavisor.Protocol.Server + + require Supavisor.Protocol.Server, as: Server @spec sock_send(Supavisor.sock(), iodata()) :: :ok | {:error, term()} def sock_send({mod, sock}, data) do diff --git a/lib/supavisor/protocol/server.ex b/lib/supavisor/protocol/server.ex index 9a865134..e545830f 100644 --- a/lib/supavisor/protocol/server.ex +++ b/lib/supavisor/protocol/server.ex @@ -28,6 +28,12 @@ defmodule Supavisor.Protocol.Server do } end + defmacro cancel_message(pid, key) do + quote do + <> + end + end + @spec decode(iodata()) :: [Pkt.t()] | [] def decode(data) do decode(data, []) @@ -456,11 +462,6 @@ defmodule Supavisor.Protocol.Server do <> end - @spec cancel_message(non_neg_integer, non_neg_integer) :: iodata - def cancel_message(pid, key) do - [@msg_cancel_header, <>] - end - @spec has_read_only_error?(list) :: boolean def has_read_only_error?(pkts) do Enum.any?(pkts, fn diff --git a/test/supavisor/protocol_test.exs b/test/supavisor/protocol_test.exs index 1fc7c00d..a2607866 100644 --- a/test/supavisor/protocol_test.exs +++ b/test/supavisor/protocol_test.exs @@ -1,6 +1,9 @@ defmodule Supavisor.ProtocolTest do use ExUnit.Case, async: true - alias Supavisor.Protocol.Server, as: S + + @subject Supavisor.Protocol.Server + + require Supavisor.Protocol.Server @initial_data %{ "DateStyle" => "ISO, MDY", @@ -27,42 +30,43 @@ defmodule Supavisor.ProtocolTest do 97, 105, 108, 101, 100, 0, 0>> test "encode_parameter_status/1" do - result = S.encode_parameter_status(@initial_data) + result = @subject.encode_parameter_status(@initial_data) - for {key, value} <- @initial_data do - assert :erlang.is_binary(key) - assert :erlang.is_binary(value) - encoded = S.encode_pkt(:parameter_status, key, value) - assert Enum.member?(result, encoded) + for entry <- @initial_data do + assert {key, value} = entry + assert is_binary(key) + assert is_binary(value) + encoded = @subject.encode_pkt(:parameter_status, key, value) + assert encoded in result end end test "encode_pkt/3" do key = "TimeZone" value = "UTC" - result = S.encode_pkt(:parameter_status, key, value) + result = @subject.encode_pkt(:parameter_status, key, value) assert result == [<>, [key, <<0>>, value, <<0>>]] end test "backend_key_data/0" do - {header, payload} = S.backend_key_data() + {header, payload} = @subject.backend_key_data() len = byte_size(payload) + 4 assert [ - %S.Pkt{ + %@subject.Pkt{ tag: :backend_key_data, len: 13, payload: %{pid: _, key: _} } - ] = S.decode([header, payload] |> IO.iodata_to_binary()) + ] = @subject.decode([header, payload] |> IO.iodata_to_binary()) assert header == <> assert byte_size(payload) == 8 end test "decode_payload for error_response" do - assert S.decode(@auth_bin_error) == [ + assert @subject.decode(@auth_bin_error) == [ %Supavisor.Protocol.Server.Pkt{ tag: :error_response, len: 112, @@ -84,7 +88,6 @@ defmodule Supavisor.ProtocolTest do key = 123_456 expected = <<0, 0, 0, 16, 4, 210, 22, 46, 0, 0, 0, 123, 0, 1, 226, 64>> - assert S.cancel_message(pid, key) - |> IO.iodata_to_binary() == expected + assert @subject.cancel_message(pid, key) == expected end end

N&xt5p=X&EdJ6c5rJEgA`Q@gs)W?#IN2es_FZcV0FHBb(Dri(PL zI7C8?A*p)4yw17~z~JWym_VpI)B&EmWAU%4aqxk)#n{kKi0Fg5YT{E*kB|CW`WY-j z&i3?kz$;*Fy<+c2LgdEnY&FwLe@BxO{0v$sqfi?lsRA^WGQ~LChN`i%_<5t7Trh>L z3c7TDOv>l^vAwBZQ7tjBGlG9`c#G3-qOF%`BPJ$hTKMxkN9F`(Wm(za`Qg0vtD9la z(2@RS!O>FcP+47Fm*4w)VcyeAw6ojFhek8{^%GY(67#uZHl==2Yyjsuc<%(33VS#` zI%SU|+>{RxWNXgNND3^l-^>tNW}h{sBotyHgFnN(F|A(9-x?@`S`b~FFL3&h zRuI!Feay@>+3F_F;>UixeFaD6>9S1DKfG8oKFpo0eq0DCS`#A`tdeyd2kKdEK1lOMEq>#GDy5zRg}%yh45F0l}04dPy#!V zHX_Ze7XKW$&Vp=4jGBAAFUIqoQgZ{_tw4|VTRZWBnNHbXL0E~eu9LJrD90ta4Il07 z@J(y$x|S|H!0Ye|e&Hzrx*uyQ^=lMF97VZj$4VjLQou;Vf%##;3oev=hFAj3T8Sn% z4lq?{!2HllSHA?-F+ae0{H!m4b<6-DeWK#aBt!(%0@?tl!e76sfvJ=QI1DfRr`C5& z1az>mA_J9VlFS*(wJ~*yM^Pb(ev6IJp~ITowEri+p$E^>(k4{%g>b%pWbGZ1w*l`= zSrOYt5|men2~mKN1VDn{IXptrlEi?zoSbC=7*>G!;x1N+!5LEk^z9-R zQYlFhpse-R1`0wZpp<8}d?iHU1768jru-3S>@A>gm#`T=5w-n81O}9HaN1W~q1^yRm#5O~;|F2NV6yS-hFoXnD9|0yRBM=V@z+5$8 zd@(6#)c*q2%Kr|k|7-S2nIv^ZsPM5 zNNP$Q;MD!E8KD9Ol>OZT3E>bZJ4F9Rh2)0U+3L2Zq=z%60*tzxqghIl37DFqN(`sKug;CwLt+^0Fjk@KRW?} z{tkQ(_C=BtsQ{m{ok$?%=WgY|bef5uSBEF?Y09scK!Z&IxLbZTDt?p~Gj*pj>L z%27=v;oV5SS3=Ebe=w`Nf{m*7FR7ph;vi*tJxI{}`X1xmZAq>bYx&+6Nv~WO_7baY zYNh)fGBD@w(E+a3ewN7s@ZeiuugDl3_Y9cip8yjD1@ck?%LxOJ=EPA*fLunW4{#zx zO)mlX=^aq3`8_=<614^wfUWOWRZtL6Rnne8`E5TV1puVLJP^Lj*ww+w8Ec{@WjSX@ z<(rh10uay4^7~km!OGD0GO>Z4ysGFYKr{m0Zj?Ze{iJ2W{?RfS5xtTb`TLO(ou9PKvDDWzIpIK4 zZ&h7bB7hem#{+Mr9#g?b2B0c3F%{^CH*dP>({i5{m>FU|pRN*p?8^h0^L!2v0R9UB zxqg(Vrdt17Q-M#qZ{NSb+{^+}C>(#WCz#~OKZ9_k&{vuBTw%%)=%|H-j&V1gW=qf0FAwms4O z*)7>En0lfxC@dl;EG#TXBCMF1Hn)J829Gt5Z(_N|>x4@-+K6#xo@~)2FVruFh%X z9&$My?Fw~0_I8@RYUfnzjiJnbW$c<-Aj^qZ7CREVf*^Ws+GHbx&A(AgjpK{crzQM( zBqRjByxQHmeG49vB&pVGz6*T+WsVy4w+~V+aj9g{m1Ee2f7tV&0QvXxTfshp@{JdJ zNip%x!M0;6h7l7XuC2lSmM6M6I7ntfh+mt`yq|UZNl`1}#w`&R1QItP2O2~LY*ZLf zh;p0Ypqe4kdm=mk?RJ-4s zPV!VQ?uF?R5t14AQ`63Wf1i`cP{-ha?}OW)KI)?IC{R(yXFHsU@~R*wN#jG20XX1c z^2PTTDq+yA*tWa)?mKQGY-EHw8fx!B2V6o3QsAnjq>KKi--c}jNJF0kHrRRPwEb|A zf>03pO@89rpXlIlVE`{1!h3rngM<1CiQWlWy}Vmns#68%8ILw!H?b7;8=SZ{LJDUh zPGkh$r-oI=2|#xtz^m^+*8?9&^)K-W{?)G!|I@GknXmuhum4$(|G&Tq+i++VWG(`a zNmW+ax-vH(u6B)~p1ckKECb(1PI{G$I3srlrMKUh3d3cJCc5C*%QMN5TT-jjS4LWS zzr$@d+xXq?R^N*Be%`T1QK+YqB_40`|EAB(H|pvEq-JxEWPH49Ul!6wY0hR-y>rLUj$UT3KG zVjX+x^H0nUDW0pQaN#yMTo5s+rVU|bH`o(RKHQ5Ojx-C~O#Q&L9F0%MqFho{CK`3o z9o{}0U+$>*aH>FL%=a4p@LS@~jtgVsI@RJ~>;NeV;mu5{*w?0vTs+=wrNm0bLS_9t zFuZZHr9s)eCybh;;nSLy&i8kOEzb9>!)Zc7sGrYC~`-U53I4j03?pn9&x-nFf<&+R!2 z=8ixR5anhY3Zi9<-QvtjuKi-AwLO~kLlVXFRW8c7A}wKFl?iH_cwd4TUHjrTdj!59 z7Wn?*ea!gtszrIvZXR7_%^;)`+pgN-aWS2-i)=QCnjNe?+NeuLg7U3k)pBi@BtBT? zY%@p4R(oReY|ZUx!@E^@+_~E8YH4RjU=_|MeGbmmWK&B1`PDd6$m3y%#Rxhk?PGU( zO0Lsr=H6-tM0piCNx(agGRlIJfWtE3JcKu=+D1)gxc9tgp8C3AVXEbOUwfTET80vv z{#xMBzW`$gQ*N0)$9RaVwN2L^gWLTcp;K$P`voF0#3`^$uZU*aaW061Yt8R>=V*oi z20@-vau!Hy%R$KYWU0eu8c*3Y^8bZd^n-&m?zVizrte+Hh^fYh_h%1Et+0oIi`hf9 zTFz5YIIm-Ruf81$WA(gGFGY91#$#FQ;hN~5qp}{?3*I1frMz` zcKF-REoa;JQpN&R&hw@q#cu+X?XU*U?nvV7S7JVjFz{mJEAgn7?b)3p5Inc3@}Ung z$aLE7f*Bu{E!InpC=i;Gs}(#_7FkUBRYxC@fH<3T}p*?QQgPMqH4 z7mW8?ebt%I&&~QadtGeCwH#;OZ+|XZ+#|8+{9^70=I@&W%$TaSh452iW_jS6t3q$r zyh~4hrylGi;X~MUfV50@zT=|6SjN2ea!aP56OB^ZYtm(?JFood599TqyF5}?{kTw_ zC1NC%SU|U$3^kx2KvD@0*x=CqqMLz-q!JB0)ehCq!2j1%_J2PGXgU5(8~Ri&M27;P z{Z9)~FOXE;o;%*FnX?*KAW^QkM)^NjMbL`a97yvNuiR$!}K|XATP>4+iCBvWt3Q(5&2h zvz<2)#IpdCPmBB5P~G$R#I+>6d8?(ADIj~8><>lPtu)O3$TRLKd3=@0AN{G)s=Y^N zyO3z_?!Xq`c0J7!cTaXp7683yM_qft96NVYTAWhAyHFHn{rCXqyoYbp_z3x~M3vM# zS*Hk^bhiix26W(#mCiaA5PZ*lE#@cbRHXVGz{QKnDi+rnih7$7Hy{Edrk1~Uf4rn? z+-)NDer1(Pb<1no(PFEP7h;0BkaxJK)8eNq?DC0Il|*Rl_GN~E$%`N!E}f}$9t1*f zva1=U4CP7?hPCebTerI~=Lu5>n%kMm$rM)I%0qMkcmKON8n>+>G{&7CZCM+W=B`CC zX3f0ce934^0Kd-^nS5ODiB=YqAnl9j?B5k;+XqV&#PVZtkdywvS-uxZ`*RTc8b}dT zs8=C^6K4HG@F%06+j{>u$nMFyLWIr@e2P29B=yf6>>5taSL`I(1E zqc0T^4p4XgoVvEn(vJ>8RAFcHwJ zuNiIum-KS@G8}&8 zlSEC8DoY4pb{_iMT?JrQ1ndho*da9R>!^r9k~+X{fvq$y!5`_LDo6iS1sTA0ARFth z%u^L$y|6>_*v(Us5hiiLxigEtvVux!qG#rADe2PrUTl`KKH~Z2PC6B~@oE5hu zRT+Ggn-lfD6UE%;9jWx!p#+4sApP^yokd8STn+aaY9Ew_@=XQaDcQqyYh?nxA>OU; zox=juKRMI+!gw|}Q=xHOoNUYlx$fOfNQ7 z*x2e;w7r|x&5 zY3u}gTOJ_K3u1`lJ5~KIPVF8Q!;*KKX}~#Bbi|+DlM9<1K4Hs6=46ok9d8*vOp83u zyXYYwz6gDbJrO#nPcX!yC(2>5Z^+vELAb70z*;5y4&QOHe#Qv%rr{-iz&aPrS>I2w zyASD+tdK8T{Z-ExQRBa%8Ez1xlmqUM+0B$2B^ zKJYv~;V)E5?6J?|T%cFa`YJ8WGnV)_(rx>aJtJZQZ~1O-XLOvdzB-2GJ3Y`j{RyU{ zy1LbiUvYN6EN3BTbkFP<)?W9He2m02(!4?EM~r#DQ*qCYSlN}+tt|RT6%TVkup0#~ ztuIIyteUJXOu`OXMHiQ@bM0jeomruj7k~+qcN?y_=y3{UVoco}pvlnO`baaM&f3rT zgqP0RNG5ZFa>nW4X9|K@Zw?7Y8JybYqqRq)0<4t!6LYl0I$S z|2vb?Qa7`)jwe8$n3U>?uTchgzC7_Y!xNL@sqQNOy9&quBVU7_st`}`#H8YI$)2-o zH9lMJfJ>H?J}Dzdu;x+vHZcGtX<0A*X#Bk&xE)QKGpi(`0yMWMNLf4sDS+fx=)#a& zJjzAL*N>)=x5TjAjm1)JAj#YIEAYjN2BW`NB1O1f z-5`>|I%hSM6exY>9Hm{96KTycej-z}GvhwX{UVq5yqN-t$0_5DEb))w{;s0kqr+A` zw`E+gc4$G~kPr`xamkSJ3HQLtP?*LnkjYso z+zVQyn3$2)F>R2UF781j=J(fOWO1fxpNkA@;=8&`r^#YAKOe>>Yuq=`FLgzIPEe%e z*J;MiDnGe1yan@nQ*nsghggBe?(8$ESmRPHXVchducKvcyZ%15Mh5N1C<4KCbM)|J zAo8>bqRRc&kP#5`}eC_c(?q?0s7$CpA&iRZCc=U8fIoQ9d8; zM4ELY<(=n`9<-BX?v14gll+c8{4G}EHZI6@K}^`^Q?^MfMRQjE28;GYe|~8BVbod#vA;bOHk=xNxV9uXgtccL zYpW-aSw|$4Fzc@%*0VX#iQUDnX0SQ=k68Eb4s>GxdX}LG|JC8#~G zQfiG?d*&d6jbogj5l2G5I7Q1FndPuAPz?8Wj50_irRtNjG#t3AbIg*_F#C;1SS3%5 zGI3m-E6)RSB|+#KEI$-G@sxS*CP-r8C)Rr`6%sc_x$GidrdSpizDf}`AhkmtMXvXB zp*B;DYzo!Lf|u~NBHwTlui&PCB{hBzK1)=Sqp9*p7<6SdyvDuF%`fxfcu4*{yGZiG zKo*_ga*KCx4CRV?g(-w7D6h)V*l8x;@m|HSR-1WLB94YLL!@GcP7$uD+13Cfdd&Xf zFL%S0UALLDDCkrwwDp5AYT^6Il}G!vaCZB}Z0ddCpnmR_sn$2=x3hCvj3)q-Klbv`m)OS|* z^8Sw;+?Sy;7&Vbb#)^NY!Z>Lg0{W`}vI%|2N%7wvacmj7Dxy<3$X3u$CmyPM*Lzyo^Kv^5Y3L}>yr3OM!oAoq?xlS+-%2BFiGNa8nEYj~xeEk1&tr*L6eWbAZW@o}dTlGzl1cKVaS)VK46{r=qe zb5_pI*w=@-sZkc*xE2YO{c&wuny*6P;mK~O61>h=im_L` z3U%}uaoT;uNoh&p;A-nO4Dpl&wa3IQxvz%c4Y`< z@xk$`6@1N5i1}8XEopQeG9uys_}VjidjnAq`l1-*?NDPwkqgRnU2CrlR*=?lL^&N^NApYOg%x9(OvCN*26Bmqn!ab^c-q7@t zd=4Gl?n=c;R>3tp!8?vF6STdyHZQlKSItT=^0bprt+g(=22wpX{D9z^>#3SHdC^rE zR<9W8AL}?7gk)7|%1_7kI%;J7Z9|mu?ar9i3o2#Hb5iW7pZc!Mru4Jm&!XVR$=Lf| z5eBOLlXs4=>tamH4=x8^3e#sV-@AVk&VgxwgnXemlAFZ;XZ4C)cm~H-W9gxo(x`G<2NG8-pOXspVX2IB6tX zYLIBC-XoB>zGfy@tpsDwYB+vygb93jZwk^R^GEO-&yc7>t>FVJSl)Dv{d{Z;fY+Pi?-xo-a0U z5-&XjY4@=Dl~bNzDb7)I(+T9i09Pvhg06xpwN!&t(eN1QRd40RpYOi>JaYq4*W7Rx z6X>g{SZL3mieuj({Jm2{>#LsrCJi{nky}Q#VwY!oy&0+URa`;HWm8u+|BXsqpei+Z z6!arO?RYWg_C@ZbvFNyD?>-KjS#ZG%kfcTJ;bXF9FhQz3Id+g|odBf{m~v4e_EoKt zn~U7(J3edSigy&;qds-7xL=?yb9S6$8EJRo+tno9{0*CVZ7f-qEX+i10~d9So~t_P z_Ns{qIWcH$)b@XqH>whJ8qYfAM#DqD=n4gYVE90~3-Y{tK*?saTmu^BKHbdD~{7KvVDa2jG0k?9R~%Idrs^ zLxIh*wYPvZAnX^_&bDNAa$`!bjier(dgFImtvyNbg?q;btK4*u)JJ)X?_!DIw#=)R{^`_Vpl_rD@?n1B>tF62U9~@TAvSg(Lj-D_PUC zk*tOGU=HE!4gfGSTU3ix<_8?KDU9lTfXttD1~PjWGs*DC@jHi&hdfNdmRIeW*e#Gn zcJYM2x8x?%$2!3%_v*{$4psa2--kg(f4T=tU?-m-!A4CuV1D7iI`_*V!Z$PF3_$@; zn;uy}Hp!}NL;>HC#DR@HqJ+FzF7RWH4)FZF!WIIgTwtdHK(yMc_?iIt5%%Q53=7Wy zgjryx{F7wsRVc^*r(oru0#TY({&7L@7S0}jedHcC(tKHRm3B}0E<2|)?;!4Hn94kF zF)^THQmPllWH7|}Ekb^?GJDst(hlSDsclP-$Qv<&jplKU~- zqtrBnpHBymVR4^7@UGQtd}o*-{I{(Hqx`pt+e)<~nAI>hMQbqR&++d1Ho>m2{zHZm z%hg;U{`uf4%K(=3lTr#j{fL1$e>GCkdcIh^>vp?C%73Lp_gq5w!v9;&M61=Bj}ad@ z)kHbYXYw@Tc00CP)LNj#o_*zG-jqx39tIC9`qXEtg<=DkMfR4OVNRvoRjeH*t9GUp zKY{Zr`Ye(1FMg@BW;0_$Dgr9yGfa1gYh=h42S66K;C==)Ah)Hx+rcVY1NMbkT(eaD zt6q9bW*)e`_?#J%%L&RoOxK>o^?;yv+OYp07rA!p$0}w^GtG1NFIrQXXMkw4zNHs- zKd8>cy)v+z(7Twv%wtTKLUsd8eU6-lSIP+$igL>0PGUV+Iv=>NGI1lN?Di+ll+oXg z4tOPVjHTp)Sf#DEW8s<}dMGcgkr(2h!>eyM#ionKAI6LQ5{;&Qxcee{-{di!tU0EeF8h5V4>B8`ZeEi z6^oq;-!rM(<=9(R7qYMf>WQG@q~vL5DLiN|dc4Hr-v%u=$TVHqY8H)*|UB|X32biVZ18qrsb|2fu_<@)B) zsQ05%2q}Gwc{yVA*U0II96-#j_e(s!*R|3k#=INz3R-&MvYRQ=l)&nurx&OE%N3XY z8@IWCEgm>}6^+;Jk=k?J9ACrqi&BN-V;Fp46C-ijFm>w6s0f>dkWH30i7AG{0`Kqh z0niVUGCxKgcUqZq3^#pS7tu6rukTef6QW`6s)IBx50mB++JN29VHp4+bCdorr1 z;B5lyGw&nNH`~FMM0r)s3-`yBO5*&p`QnZkHA&|maO>xUwrI^q1u_rr<+0bB)|MY} zzMkL)X!l}Is>80DL}x|xJX|RN$*1ut^TzY{wx8!-=nBrLspDA;rBYuLwYM-G(K;R4 zCfihw&xJA9%@G>@&=%`gZEnU@KHCMocxPhv45|HD_6G4qvRx4QsdJT6VlC4YIBL=hsjAf- zGJ&A9NO{CcF!4(&D6?TLBERE`+U8*3b04Ghi-!Z>3um;|v$#ZW^^7+|RIe44gl=A; zNQ`7lsIK>7)qEwAT4C4ufdsKx1r<(tulMNc9S!~7`*EXlwA}Cwas`L$PebBq3kkik zg}f=QK9p^sO|M_IUg~lO80}^)8|#wiU(c4oGR)s;cRS6F41XpGSjBu@efg`gGGC~! z$mrg&j~<2>g-j67$&G-!m#;xM(m7C14LzEJO0g&zlmAuI>7H#=KLBiVN0upNULMcU zT8*zbrxobVy^Zs#G~w=kxh*eGg-}nt0N0C9y3hCL&VD$NbBDs{h)y(M#cMIXi_4A0nGlQTs`&k|9U*#9lw=pcZ_7 z)<@tJ2DKd2%Uw|U^Sf?zx-2A~peRVZU02GPuJVo$CC`<+8hGY}(ptKnWhe7O5NA{RpL5~(5nY)rS>cFUhrx_>~u4B%Y}9a{vGe5`ofP7$Z}VAp7Q5% zF|V~7Ss`OIH25*I6L}#JIG-T6xr1bQRQ`zb+1!o~9}ktzGfICxb3~``ubYOs-V&;5 zZ+_h_y*I`L3TntKuA1-GP(|@qgHff=8{}BHE;Gekk-u1)g#fY*6>$VNu5FWoScnby z9tb#XDm!Ch4VZu*xIh6RF1`@mzY8ko(XDnw^)|EZS`5j9)Hbvel%CD-yVQT z`!Stkm>Eh^vcm+b1WjyQ9fbbogNgT0J*w}#Bbf^agoz31=X84&}^^3nRwOhQe< zOTfw&XX;nj*uv+uhgbMvV6ARV{Qa@SOVbp zF2GJjeTldmWUEZzO;Fl~Y3eQ2?e*XkX)|jT!`(rGWeH1i60~&RTvBqp)YnUtLlA0G z?udzahwgHu!?-K{A%r5%^4Rg#<#1@_LUR$hsG&#Ke0Ra zcg}9+abjq&4(hK1<8skRo*&zv7Gt7*|lU2t%erPk_=ww^Y?%ssj5K)bUP~x zatjhga$l_=*JHVOvwX5F(sWU|<_MMU-r2m&Bt%27bI@2v>h~D*85vi(ZD5Dufu$%+ zM2|TcEh;s|=~a_5roDW%2Xt&vxu1IzBj4nE1aGONiC0af<$fOA4W8g8MN`Vtn9jW4 zuBvyw(9-Sjn_iguF}<(k?$xI+Mi@^-oMQp+pY0<2(3j-pKmd7vCcph^$($nXWYKGZ zu!oB8eeip0V83<)&616ey138nA9T`aR^s#&3(Q}Q7!^;kYjM#6JJLCBkvY>n~pUKCg!}Q&z6QVpNz>u0(OIXT1 ze}~3q6s!09DQ%qfQ2A7XltARmapDulxjc4X^3W}w8pv#;>HsLN!nossLrlM#nbItA zA0&^>gI-wbH02*1+XP4&O?kMWaCY?VzKz6;RmH6;!Hw^AzzXhZRV6l_fD3xxVKp~y znM}5*N0&PaAlcyd$gl0D6aMB{>mA}n2Cw^=OE$yEcqW&RL90byM|jN201|M zntFc8Zromyz<3Pn7^I(it1@LjY%#_c;fGPE;wLIioPVF6$w+JVNDCo&&sa7Ylz{Q_ zk4x5P9*)x=MNf{q6aYh@^3=n$RjcNRSIrV}o02)7jVd{SfMGsotO#hGbuABkcptF1 zY|QWRo9bM#{yx~z97G|nwH_@Di%vI)$G=*F+&m{bNh!EB3!c#__cAPeC2X_GG(eAo z+3Eq$&?>^pt*&-2g*e(T6Q^lyvbqy;6L07hFXoeUKOU-48SC;vZcmIYITh+X$co1qu1Tf@Tu80uix##^NK*cd;;t){TVsX~L9T z{LK2K8)F5PP_24}=6mwgyzn~TD+fCL2^p95e_uV63L{}=% zogv&W5*6cL?c7A);`BsyFr6yU2+W%TH*_?gjfeN%C%0b_f+G5HC2i8H+%=~KKsEfnh@MUvG|S9I>+?v1r)>N5Qr!;_8O`S?a zndQu`JO|IX&|j_)4;aSJJ+~Vp%iTi#{S%B1^_|vO|J7^Q^z^iV<{#NoY}KJ!hFj^) zrulSCAnjDbp5-GqNIl0jp8Cd2GG7+tSYBZQYQLiV#&9QdFulHC!FVg3dDZ){PB?R$ zrq?Rwy}=hv$)%ZXrKK+Ik8lT4N)<+y3MZw9)vI)J7&XTE;2M;zFRDV#oNK8v;7(5`60=qUG0r=V2&uE({MsB5wLceI)k~c%{h?aA zL0G5u9fRDpod4%vYk*6Mz?p2Ik{J*boLLge`6%Fzy7Iq!U)l-U$n)>>YyJ-9^cB}G zMS4s8TQe@ewYt_!m@!(KyV;=Rlobm|23eH)h-)czcX+Xht<;V)7oY+pO0W{M8!N|z z;H+_!33@iflyL~(eF}vvxb2qoZp%#Ful6*Q)m!%x@%KuqeLh$gIA9>CD3T= zp)daUV)cN9SGk#rL?YQ z(vBzn@5f{Az6wGir4SJ0MD(&pGv&;kC z4cQ}COqoP&=Yyy>2CEBWm_}bSbx)9DyWOUQ(i*Y&we;_a-$Kkbfc~zD`+Ph(WQtGn zLxb12iczZ-x7yh~Qx>hKxJAR|QuzcH|7Lx;P9~G!G4Bl~y&&@oz$IfmjZ%YM`nwa{ zm-A?NEF6=fF^pryE6?3)`K= zsQl$OC`?PhABZ?^MJhVJ5HJVlJdB03&;hLkpD8h0z0?%c%` zf^x;3S^?Vn*q0Ak2Z9mUeZ_`#Arp<&7krUu_sj;d*>_IW1H5%sD&HL!XUfjv@7Kb2 z(z(s`_Ld5p$f&#}%NWg!7>tDpMqB+p$+k{?BH`{m9=FzU+7u zi9<01F=J_NiW|&_eZ*Q3@n3+=i>f%|YIv4P^MmQ=cqXKN`Gdj!9WFETO3+5+O-sPr zqU)W{Ro#HaEdK6gYt#xTs=T*v2xU&M)J^VU9CK;$6K9ubBl~sS&veQbHuqz|X*ib9 zpHH}=TQ-fKSH-S@P=CCa$`+tPpbbO}Na9q>NS-E9Qn)A{|?;V&Y`r z(n2R|938OKo8OVM;FY^1&>P?HMwa^HgU@k7Mf`8me9-D7@ZFtEn%ept7w_&lVR;#& zmgX00oAvgS{&sVXF)gB=19+@@P@HLd!(cT^A(<`-a;L@DN|)y3X17x-wc>kba-+s5b!MlZ-Kq+JhmM)FyulU9A~f!k{D=zgIZKI<%C& zzk#}~WzaTC4-aZPD-6jPtDd*J?lWTUUBzpmo=p{v?^D~;TLiJAssYaGLYG8)=2K2o zSq;rhffI%4!YWzVnF`4l#9~Jl)9*oA_p%=!Zzr!5zlF=3Z50`*JR3>i7wI;i7G-Aq zgX~pN|59uB{OzcVZ@%7PvGlZX#QRIfZ0sqVWgqOEsMIBuck4bM9x-&k`mDPB1Is@qH@iz|E^`XJVJFscL%Wjad z>sde>+*IBDk>Ywj)GabRvl4v@A$Kq1%iFkAZ%X}QJ^hO8#ioxqIwOJE@pw-rqQguc zTMH!ONk9IH%52@T5AEr03t)k-r8ASFafPiM+8yo)k2J2FSA~ZuCl7t+ zFC+Fon4Ev}X8S1iwZ>ny6fSPdp_Gs;PLN5-G_9aart_~Kxk$CeoU*S~GoHWFz-nMO zk+{y7;Etcwj3(Y_M?{V&yNQbR>%`iib2r+f(T&k zms`}|SAY`X>e$;0WxCUL{^;;CVpzy!L$f2@=t$M;MT?Bap-H1#Oyawx)xsHck+@c= zb2+tXw+mXEVYjy#Ie?wpwpMUn<_Gt!9{2m_ zU-LPx#3&_|OJ4VlRF%77@>m<{Re(#z*&;qg{xE@3G{aK)ZPMQCD~&fTodHodcb2-8 zqw_J|{BI{L_7<%4U|E7!*vl2}`z$b@#SXf0Pn0s@(QA8Jw$zz|Th7LM)>I;9jVMz` zc%_fWT`dT3Voq}jiu?y_t;9Wd=43W0|9GB-U_9zIUoK3u#SCx0d4cokYKZXpR$tt| zycAx5mm*0atRU#gOF{kQrQl*V`wIN{0UV-AGWOcY`*%Upe+qyaF3JC?K?2|`;%jQN z7=Y@3HHCu^*oO=5|OU{NWSW?6kV$z|?MLJK6!XyYyK} zpG6JD(9mnCNjvlM>a-Pb%1r;M_wM$%wO@Ltl)+a+_D39F$JotAanmlLz|r1w+$rCR zjC7%_#fst9|J;#SPoeR*m(!eovccWSI*gUELRg^CY?L^6aJy=(TAauwZBX9tXVOlj zPnd2VefZR1gmrATHf!V2f^WD zJ{Ufa54IPqP@4voN&{>NcrC`uGL5w+2;LIzx?F9w0|ysa7)?P1!ozkN4`!p|F_^7h z;A{DFeqkyHDaQvrb5_F|5+L+Af${!t>m+~qKE}8{-CA}9rz|X70_%IQucwb=4x`aZ zn{%eFDeS5xlkHE2BMLPi6s`$U>XbE10Pc&P#D@6|Sh&nuyjfn+eoWi(y*GM;c~_qP zk;u_F4lH#Wl4duuG+L5;1&2%+Mi+%c_w_lxH$&z-4wmy_@AcqAif0d5())S_E}0^K z%)0m7OFl%-61>eh4s|NMNTJg=wOd?!uIH`(^JX3Q@a3YS#wc=4!^iw2ot#f-3nQ{H z3CA#~t>Fg(r9x`9d+`qiwO18a5;ZvL(8Kk3ogFmDYJ8Y3*OTYs>3qh#%U0e-xhZlt zSMjSyAS!>M91A?o@z)D|3bkHN zHqwapi9QU2%AcHnhp>$&bMFeyxu_cqpjQo25SZo%?ZCO&`e>odzKu>bQosDKn$YN8 zu7{_PHNYsvNB%GNzAB)OW?47528RW~NpK4m+#$HTdvN#Q5L^>1xCM822yVeGxVyW< zoyGq5+1DQK(|tN`Fui7`rn;-E>MQB(>el%5=y4Av&DRa!m~-;t<~UOdVtR<_CzaCX^bpS` zvi{7Tu=wfV8k;3W&6-u)gUVt;TL84`=B&=6nO0Wwa_2f3Ce{!blkO&-2ItlLYmyC; zQJ@0BW!6F9AXL7msLZOyFIR?|Re#U9wSZVMmCtkbehwyZ47TbZP*B?!iJ7SD(HdUW z2dq%GH;T3jz0W`fKC6YXkHsiWqCC`cyDjC@*@K+T?03q?ai1R_2H+8E1J#*WwSey2 zOiDOypdOWPLTF5VkGG{&#kx;8%9V=rO+CgBf>8VlJ61ls9A$d-5ja)%xeWK7#v8>C zlUiEYj8>lIQpCA25w$Efhx+^Tr5S`^AR`0){dgcgF0jQ+q z2mUUioEy;l119DRQki+DsjC3*eGQw#(STi6+6wN3(RSRzvJD=rldIql7!=Y4$6_4O zp7y(|Hp$OB)u>%-Epa~uf2Ww1{j8M!3?v)@>8s$9)YJ_!Qy@i~qyk)%`u7a|0BnDf z0Rv=)WTxQ1fq|BP4}kAz{`Vbr;DLrp^r8RtQLgYJwBz-|55q|faEhi4TJeh>uE*5) z7g&f}AdiiY<0lPVz|mT8fQ)52QZg196@xV7eI$WMCUo?2YJG z9ou*f*k~0?U>B`4QRtwWZuJo5l+bjVy_~JyP&tcBsy^rl0-$uAB?uc1*eEU5kK>Wn zE+^T`^iZb4Mn-4pgx@YqseLF>cS{M$WFW(NtB=8ZOrWc*a zYH$F7+kKO(oCkVGF4N`1N9HYING1>rO`Y4>yNw6MBDR7O_@-qHFZ`B!NRFmtyV_KD zPTz|bO-&v7Kv9`jZR%or34UMzoNuyqi`tbSdcB;ml4pdgb*2>`(Rz0z>?KM$BBc}* z9ThLJY)0KYA0PmFsojz7WVXV94OM`V!SOYw7v-V%{aH8W^SlI_cb4+I=xK7t%@L_c z%^IWHJ65N-4IZ0sfznqH#ecsTc6Ox`YMJdftTh$Rx7~jM(P-|wQ_yD^vgw95F@<#5 zt0Q1|Wp4gSllvY#>PU!HP{6nK=yh1J^&WH_k_iCvuY0P@CCsr`JHBve?1x;V;Ooi1 zYb#VKSJ9{s$iI-KxwKw$et5((>TKA~mY2SIHhP)*pI^jFf#Sj{YAd@&TAQye+EEhI zi0%#Bxjj>dP4l6eIq|Tcnv+M{Zi{6^(|3WieaRlD`Eve=DEfmd@@D#}M}cGK$KMdW z$7gr?qVvl=n@=k~cqX$WlY(|=UJpw9Q=~PXb3B@jhHaGMQ_gZCe9AN`P1%ZVBFFyo zU%sjq!l0XS(Gn0DPelF;Rp9WL!9rYU8qslenFTU5C9<;I_kenfFp_NmY9^UpZuCeI zc&?$BzkNEqSliz~lt|lu)WKd=OG9(hfYKm6nA@R|Bbu0FY_5 z;toTr%FCqpOVV}Kz5T?mfkLrS+bLfCqvRfG2Hj2$zF(9jUo#j!KsoQtt2tXrUG^Xp z90@&PjIOafKgTDtdPP2j$i-4bWxf~F_Q^!FZtWnxs@TQ=PBJ-jM_RY-k*qBQ@=YxM zgtj9qsbewxDHeaZyoo}E$h0!fT^V|$YY&Vc&F$ua97%69T&AfMc+D`K1t*e5Mryxi z)rf*KOnX1m0a-N=JK%w+@YL4~Q*g_Ge-8k69RJrH;4B{fUeOY8PU?RirA>-fDzdU; znUD>{^fnVmo^y3lZYD1?cGL%INlPxn4^k3rb55mZ3xtj)l*?6QuGX)^8V{6Hk|Swp znRmiC4}Mx<23vfpB4A8z^T$@L*Yr}{s31GBreO4bdPcq6T{AjhTS%khWS)4?bJnlT zcdPMR7`@#hc5XC(xL2;U(GYbZc%I{+t^8YOZG1BV71ZBRW(b z6N+=wGPW4F(ou;yf&AQiiFI&ZRa)xCUEM7NoL zCLfIB&8eEi8)s*TZNHB??s25%v({-%v;JvH2>D@6Yt~k2&LS5Z0R)<%(=Pv5G?R%j zYMwp)QlbRCN^?b9mI@ghIlh zmN;78E8E^A#uUSZH>Dye>Lb8e*2J#30%9HT0l{-ju+`}8QHc`=RZU_~Is627RH zgb;9Z9frSN3gxJVvDTVANrHop&bl|KhC5^BUyIcJjMb~XG5yIHLZG5V^jPWCYp`k$ zF0&NTUlK#5NqA-cRn4S&b#M%vm_(~uESw7*SY3Lc#-wqdB*fU?M)E0tC?-^vkYAb~ z`Km|h0Baj_TY+7M?u_4e;~e=idb zF6WiE2$~G4DM7QUiRb|}4HQsD)<<)VfnTHv%eRSO{U#X8jGnby9V8+xFM}0;a8uMp zqv92)KsR;<9l)nOw69JqfJ1CnmOMZ0mmjZS1hd~gyQir1@Y_kI3N5+v-=|J{D2*dh zxY%mcE3&H{wCZjo#;>PXdsnEQR$v~4w3*6|jT-UzYKnImL1y|xW20W50O=@XzNfoz)p@H{XCH<^~W z*8HK2tcB0}@n@d&HtKs`>1=q%`PoIFN&rYlJPk%kpV%HQHt&my3{!@e76=RAqDKsg zC!x!iz!V~89zcdcOhqp@k&s};!+J>9c(Sq78Y^0{0--1~1{T?6Hlw0)|y z(n990mj$*{aCFM=LHvt7pc^5y(K6j;Axl;I>~PR%k**yoY-6*1Y~B9tY-m^fJWa^5 z*Zp&*HzS9~^@pcVTVlD#NIrVz7g5O`r}Zz1OISAZzY4-mBq##raR9635DMa5x9}eX zvFvijYkbnr^FXKfB>-H}*J_qED$*<^T55Jpq=>+`h=eix25Hy|z$~SVw>gAiln;^X zlj)8KGg}O8AmJ-K$G0Lklmv{6m^#>|2TS&R9CcSBmpy-%ntXC+c$JH?IIJJm{4t~n zg2SHjB54VmRr}JMVM^{Q4P zz};JLbrrP$6$1$cK-GTeaIX~&EkHtN!i~Cw41j0DBZHg3d?x5d1pfIL1e7Wpxk*?9 ze}}jO13jL|-H`tb(7z760lNpFUmXdA_|z(TXg2_2Ej7!ar#47Nil;+f~dZ$5!&jtr{qQb%(_0yaa_5MYu1;Y zuQ9}r4c*ZkJi`qH?{zJTyDY6+Y*#Ckyg8CB*13nSUC<*1G0BoEEsf5s_td#-kT(24F(f` zed&kZg;hShBP);$b|F>}Irk^ODBgHlK&?7A<{4C*V$KKPCSjjv9wi-?^De6Kdc(Ax zy`IE_B2JWSHs$Y_)nPUhn-T!>Z*=naH{%*J1h-Bvn&@C}BLp$Z7ROSST>FXafFebX zJ(N!_X0Ytp{6VH+jpvAkCZ7kkk*O75rfnQbt8V_7C#M;0Y^N!c#-a+fVqOecz{}a} z$)!WM8vHMd2=|v|s3wntG7FbP?9RSR;*+9q@ie`b1!wP28EG6rupn2m_#-MW^X=(X4FiBX9#W@+{0S|MkwF|CdsR8q5juclE;y+^&m=s*@ zI_Iz$ZFnHps-5m=_C+13^p6*?zs%+z`87W)jH+dyP?r2o3LB84VlNAvtj};(dZ^>q ztoKQjj4SN_mPg6cccO9jO;HO&_2akX!$17qS*7@Nzw#7E)$$U`ofUN!O`N;o9?5vc zYk#8?Qwmg)pPR3f^%n!RsixVhGFP4a@VN5sLdd<5xWBJ894xoqT@?skd`#wRHoX63#&kWl^p+O9CYMmOP`?>*cC~s@S)||7a}0@=W2@6Wl+2 z&QI{5FMq!p;6`p*MY#CEW0p0LF|O~6iQ zc_5Axzx7VlHJ8h0{gyP>VYkMSZ?Dj&<{~geJx|)iV(M3QT^_Wt1CS@NRwX6NE3F?Lpp5K=5J!f+r5Wj1o&_8aaV2Vd0EnynVU z*a%OUd~_q!))=iCL(vea`Q+qCTFNv}CjyYd8TlUt)gtPV+Y(Xhc0utXJ2D zKaoC?;MoM)V{9hH^FZbWI7pXi0x*8gnBFnk9=ZlavtQtKD|o6y5bFwb@LJFOp_b%r z1A+@voF#!^obC>jGTY-B$X4CPIi{wns8&C7m@2tDu-dj%k)@QnNtCrU*sL|tM5VbT<}8BfIWW1dCM!ALH$L8Dec*Kzp0~pms=4>=TId`r^qGTce8l>(YzNQmsAU!5IikPP-tj(lgE`_n^ z6H*?%nn*Wn`HX7cg^$+Yx3mW3c4rx9p779)0~}k0hdt)p+-Nvw)n5Y!UcD20IXKSh z&f^2oBB7Yf_-sFoC<63-w`b=m@{w{4sMNu33PJ%}Ps8M{AIe*8uVkiJqRuG(3;d2t!D_XsINq ziSMLdw8X|bY0@RYjbjY4>Gwf?IpyDJ*rcX=DHW^%w+YntV$0uc1`A&!URZpx!aSW` z>~nu9EN8fY+Cq*?#}_JJjVtv~Wq$7m~S_r;Zok$jMZv z<3AD<^;eaarXH(hK2x-cRdluP#k4x@?_{AKK7Tlvzcy)Jso1hq&axnMw%AC{W!)JK zF%~b-#%9n>rco)+pE4KJ6oL38FQLQb6GTRBDa6*FBBNefYdj}83M9frnttF>Ml)LQ({XR>+c< zJn^d#b8!HOVyt+%e2;~*`O!HHt52_Okosx%8GhF|-6TDssO3jC08S&88p~R~KqG%HXWL1)tA# zTv)jpEQd04lP*DWv9K$+!eJLJ%x&o&dG!uAI^#zReH-A0lkzt-vr$8R+Z@=(AE zX3nYWerx8sjvFX?(C^fw<#IAZl}Tb9y#7J9$!2~1E+bsn=kb|^&^TUrHOjOfgM5g# zM=gSBqdb%$wEOseHo)2ag=H{?328W;EfA<3n9N_Y5nT3q2^hu3!zLE-#V}s~r2zz+ ze%g|JpDtO7aj3LlnBRrWKB~-tgCA@nIobb_C?O$}T>D->6!K-H^6Sz5G}7Mmr;~}T zA&^;j1*4PmM0aC2Xhabn^IH~GIAowa+U!h;ZV7yEQ-|XywxPd+mSm5y%O>d~jle2OUU&QKJQ|F?&K4;{km3E~&{V8ukJ*v;dTn&9;P%iggoRX!1#9}_eJ9kh7BVpW+ z)&~KnZynxG(V`obXkq<{{zyS+n4XPWOKk1gL3||7GiQrMEHf{>^3&)i_sjXoS`&}^ z`A>#Q&8xo|ZIk&^Gjm^EQ&C;s-U1E2>TNV~;`#~8H*gz-shf|HWT=($tPu4|+&4K& z3^sFAXkQc#IxAaW6IE}BIridqwQAJyECxPodU@0rKc+;^ngNxa({NI4hqKco1pm>Y z^%r+4s*sGo4{%MkBP3{iuox62dr)?w2$scSy$StvR9;o+Nl`qK*_`4~pX7Z6|SQVg>kWr!au(Ys;jt21kGLz+s_D^q4bn8y_klEEGXT zaRpYXz(C7?4S;|Yr91!R8Zi9$6AUIwXEPch0Vzt1fZ{!eVKc`(gm(i#`Ei`EVk&tC zm{80<4xjMQNcj$#1TyO4{2tTz7C1=*r2wQW6{>B2wB*!j0fkNt2yB}6n-f0y56A^C z{_;g-mCUrdGBSP80&%pSfBadYg)+H-3cNVAO4M^k<*%a|Z6Yf^_pw#Nz)@fC?G@{S z5zTWZ)w6pLgdO#+Re zpJcs1;i%LYwG1CumxZy7bU<1K+y0s1Op|$XIl7On4(JvUCE5c;t!{X@FpPXcf;gfJ zzff-Bru^btr8nLGoCW+SrdZnNDyuX7iCc_>WzCdmz^)tC*6SiHgIgyUe@Ju_ez#VV zDKlX#L9HS#X>u@Z$&OddUDE5UMtHzUFydvXxG=&gy^nTHNd7pX!j>O7AMG5=VUCfD zgX~{rEhz2FG=n!4=IqyQ2Dhk?`S~&AKtP9{bk?W8GEk_jeOve_kL_oSb`nE+qexC0 z!GX(LqhkXUiqSM*8)8Qy%z`#j*l~q#y9anG1$j-hazBmjQ&d`0u}kVdNc>>pWG_~{ z^X~tJU$E4q^i%pfO|2qr^Kdr=+P#$&A_bP8@*6UItaB@*{tgmazsohtmt9oG2;p2* z&I26UYj>?oeZ#yFpzW@9_eG1i zTk1B*rvl*v=_))McMP$0d;**O+4>!D+TQIO3X32r^2OLbR7csh7Wm- zety#V>df_cPHqA2IbENG`2B;Ok(fR@f_zvze({$=$E&ERluuk8y>8> zIjIJ^TTom4C?`VdmeYrJfC!1EnujYG*#m9)sfY(f?BA5yE>Epp<3VWq=4Qv z2J00d59MEL?Po}HDv?Cap9UPRtGU7T@eD`K_C8pXVk*)Zf|KBrA4eXaNQa97itpyP zMwkqp@AEA>(WkVe+#uBeYfCp1MhwRhE={0PR|p*g^SayHsktfFOCB}HE4;qje1+c zjQz!CaC+k?gVk$ljD^|>OpfT>+2v($(%+@Bs#0$GnQ5n-3^3u_>tIGNBs4;ap0XFe zsF?iWEEu3P>M>so&tfhNrx+PdW=e`8m;+kjY_EC@@gkzOi=YxM?>x*`uLzF8KE0py z+YQHE+U5!IKtG14!1VUhFK}!jg43dUp)sWI>sokdpnNnyJ(hDD8N2OX+xc$yTmXBF z{=4rJaR*9?OpQWg98=|$pwAf8l~ZWZEtjGHPcqet$%K;9KGBv{&^DW4X*l}BnI}&0 z%g$f8`>L3ZjnlvEr8lU&LCaq|^=7+pfVum%xg+}LEn#qQOI^h$CniJw`CI>XqM_KQ z&!-T3Z5JM`)79PQCXRbk1380zZaly~8Wjx(9K4#pz2GYCQa>y={8LsWg@%K0*% zu$%hQPl@}|0VQ>4-B;VnzVc^Wn3swz{t}Z9V9f3vfZ2(@jhYAtT%cPJ0Qo)l zXnh7(XZHj3+_dMolBhsJ)v6xwt2>Y_o($klWy1o$QV7VoBm>;3XK-@sOGUzWui+G|WJMWnGoXJPfsrP`^kOmm_MWN{qG_^lVy-%VSv%kk!1 zKv+*}uNS^0w$h8~WyA6sS+X}iyy-RV<5CE0HfpcyQ`NrvRebC6QWs7}$e2`_sycil zYtJCHhsFiMvEh!>yAIqv0R6-^5L<%z8ENLkd*9btfCtk}h82!jw8JNO4YQ$4JJD#7 zHIuf#!0Vmd;P!k-{f;lQe$c4jME>?lS%66R%s4#XJNukVw;RrHI(xg|OU%MC@7*dc zv0R@G+{20CwGj4UNLy1RrIr(~UVXw2VfaK@8?pOBVkdSyZ%Lo&1;cua_+WR%afa8y zbc}HOeRotz#OJoZu6jXo&U6hL{T0c-j|*k2T2b__o@OVBHRvo~+E-4VALOHH z+u4Hpg^wM+11+!~jOdP*s%6ln=xK7%eSB&Nhk!0O3nDkqES~Fb&ZARTyarf!&DI2&j~Ohn6`@_bzL4}iu5P>0ih@slQ--mb4vuk1I~J$VdjMR7E7qZoVg44b-ob2j5l zUbG8#B^JULqk7PI+_rsu;TN^hnVsVqo7J|iZC_s9m3dL9B{C>K@~o>RK4?_f_+&AR ziZ``z9Dr1JWz6qFh8EvQ>q17N3RocmR=~1UwF>|F7hJCiG&oQE0|j2})?ie?8XeOE zuxEgE`v;t!9TExdPY|hqnF8at3T{te*QeJ4f;M~)m_CL^Q{CG?KmP`Dcne%Uc^uPC zT@|D3Z;h5WqZsw5dGwSN5HT`8!61#ZGvkWZ(d$kfF4jZ!gcGk=tI5maKW%a3GjcUs z;|_z2nxEQ!#{vz_S|=Z5YJ-9=-(^j8wC;%kO|&wxoSiM=hlriiB;+6heBawM=_9jL zk^;dL3Kxw=*eZoYT|F9dW!E-;V?GCcy*B=4%8SN>t`tjPiI^RbYeg+iHTLc7Ki@6w zEVZhVUBME+clHn6wS)$-Oh<;uKxP?4CWU(D;_HZ4^dU2wcb+M zON0=-iki@6q=g7ndHc`X_u&-x&!3!;3IDdUoC=_a11D7ozM;vPwqzwi`>=801ws7B z_Z>SQ2{yks}(CTatXhr$|bpk+V5DGDKZ%fQ@TK8?gry&6!P3J$cTmpYv z1u*ttWsXEd?`8d{d>n&9QW_kIhPydYx{VKie)p-M58+F;tre;#Nf{`{i+=RCoFTF< z>3uKmr8hnY#dH4;YNb!2$}OH-ki6cP{kCm`0)SA9kf8IA=87kU2?<{)h>L#>12GaU z0q`(`PLlv=BqI;#*;edRInJ&J90k_0Ra`nTq75%{^A$RYLpHTRde;ZTh}fM;ts8O( zABb#WqgH+f)+7(OZ3)X~MR220NK|mh3xruI4*atS9^mD30QZ9g4`}^+0Jx*^zwST+ z5BwjGvJ)Yh!So^|M4U1ry@$0h?k@Wq0zM7_pN;hX9%Sni6$I)@qNZXDdq8jkt|081 zz56^PC88g=i;s?3ofZm+tw1E!7PZeGi7-GS+&&A?-fD{b+Q$_E)h2ScMrP5x?4JxG zfA(upd{fXqM~Bde=7Xdh7I#RL2FKq1mwoOHu)Y0oe=RV|Qd zIoTb2-Q5VkBpt|?dHkNpU-gr}cRh*4KaV3J~h6D9teuKzcF$pg9Bn3VUHJb2(nywHB)@5kD_Jbtubo8#t! z)#E9+xB4S;12VX;0f3XAtNvecas=R4+0dW&M1Y4ph5)U3`TgmIf%;Q{I8tCr2Q%ZM zz&pAect^(^V+QvL62Sv20c6QNuv~#}Vfe3Gz-3R8`f;-0TcW|Yy#LD_XAYiPb0O-e z*9Eb78bCR-UidoouCG)7x~7)@pVt)7aky+Xtlo_l{wD+=0bfG0Z-J@+5O__|F0Yr8 z=;HtiD~bHadN2tZ4_E}!)x-1seKo*~H9!0W?6i{mK>H>4$R!|gBrspLvJ>VCx0DpLPmL5 zOZD8)6`7>wFA49e-K<}F-F6hEB8H|)4_!0DjR&MWMhMRy8kpEtXL>BB$4vHB!>KX- z4Hrwp)nEcM{5FmKs;(dhQq5{}J=dgb>`wyC2)i_P;Ogj<^J!{YayMJ>nVKv27fD1ryga{BbaQvU-`hCTH z#vIeJNLMnp(ITu59@>xh#DN)Ho8@a$9FB?QHwSJbu*FI(zSWcCE;a)rP(>#Wfhl4O zxD%BmXwvXlG%5bq){R?giR>p&{2rzd+f(C#YXS5XWfcw0AlI;FVd~>!dBj||?>N@u z>&M56i%Uy{30FNwFQc4mK_60zq!aWPk`nvzov*ee;Q-c_x*< zD`J0(iK`CH+bUm3$a_rzV=)FH0lkZxHs7}GS#`;Zbnw;B9uQviIn};x3FC9)lIUZz z=e!J=<#%kLvDwFWQV7|;tu!PRsjb%~HBwj=wlK!WRVUB!Ga8>EUT;$j^KA>m*j_^1 zwbb<@ckNRW5YX#8a5S%^(mz1#Cll8?5iD${T6F^`*>dw@pvQqZ6og1>KFk8;zRmVE zq9(aWFuJO*QN^l&gE3yN3f7JLq^=2mbZJ9CJS9K}w>ChrB^|lC-0KTl!E|Z}=nd^2 zn&rapTHX z$oph9y@91OEVQxNvwLk#I>f(CkX7&#GamB$jEEfR041Ed8B*b*5FYXZ+YBDNqUHfq ztLk8et3T0~V!txH{*~bgs=%YAXpi-`5Oyt}QU00X8StD9nK6Rt#2K4q=Hk|*2G&Je zADGT<;PrY&)Ph(QYB10V)wAjGE}@XurC_cswNG#jQ2{J8aZ&}=jhaGdixz~}aB=zO zu^S@%F3?>EJl$405E{_t2fS2QAQ%EUh@^Ug^3mL$Y+qg~630Z|%j_CKEIT?d!%LnM zX2M<>uJy`rDO%vsx!A51=g^mIKEeOY@Bw(vOlknqnYn;Vcxi;CSiovo*#Oh&2)w?$ zGL8iCZ!sj~(46I{WU~mwgDSw4hhshSG!rljnYn0zbyG}buv`jA!L*u#D-R6xrSre0 zyROWa&cEi0DDFNSGAiHqk5<&|&V-@DZz8Oi#v!7w3iBk090_2CtNKV;{)^%CuME${ z1|B^T#;xuS(9z!Y&kSF^zHt7PPMFKuh><8&KwJ_|?7?(4{!0kaU=CFv{I(8jO>=d! zeZ6P_u6W&)*NiX&ECfC;9#}UjaUV}q0V&w0ZR7@vEZ4uLTT4v1R!_Y z^#8~mDq{bl32G~qH5v?o6qw<~?YXtD48KSKGrVI0{AlwY!V(^Uj+M24&2Y!p*B2hW z(h1%cOvPvcaT&U40Mq%85Qacr7y15nxSp?`U{Ltoh7E9~?Lke`6c1n_@OhHJx}lN$ z=}Mbf?tT@&NC*~L!hcP7DJ@AjiFGSn?sK*;Aa@rD|HvI6^~bML9|9Sb3ugHA%7o@C z!-0N!0K@go!H=$4M^Q}&=s2Q1nsw zeFm2Lf8-93de>K}7lG(g1T%c!*1VKTA7FUPE5j>lz>n6nhohAPrpxt}bN{`~{`WTf zs!0EPoBjW>%?=~nZx5#(Wsi8enGluf2!*8=nzXO_&SMx< z-Zgk`ygTG}`MVzvsFQOYeVW~c_5A%9QXQHGyF1vD<(h?a?C9)nNcU>tB*QyWst?H9 z^a};p#o{PF94>cBGn`Tcz($G?pNs6FKqAWQQ=m{u@aIhgIVwC8?51r8e1#RrOUy}#9+9oV%q z@Ew~-J_lqqE!QHLPz>&l5&FK>nX6!??&q%>0xRX3{pm(j@-+F($F3(j7W%F(24OlazD@w1AKOlLc(~+wv;sSTG_Gjx>N;;GB+cIZ^@i#!;+v9K(A8ZV!xvGQGlS5|AkfR%!B=ILz64PXOg;dS_bj4G9OU z2(7La@!$8mDPz>6$R|t07?iTe;WK5vu{)y{1KS5%Z}*h)12-j}@OIsAFPRowJm7BH z8G@!I(U9OE?|c12K7TPl+&$RYfXVSbK}1;Z+o>GisINS|=GM@kDz|j(Pv#^f;Bvv) zW|&63r;yx*Zmcr(ceCHhLOL1)JWg-I|`!8EmJA0RziLK6f%L=zEu2RQAzsj?Cr>2GT zTq%J-Mej~Be(m<1)LE}Y!bL^Un+ei z@QV7R;$B!nb(%({FO^x2$){DIo`W*>smSiG6R*Kr<7%OJ_V?QA;d(aq)qc-Kh0B~! zsbi&=e3g|bl|~r`bSu1XlFv)-mDgv?%VD5t0|&V@^v%hleOL8G)jY~L9dqb6S;B4#=9gS3~zd=v+amogyXmG zEY%q+264JG1_QvSdyo+QI`^wm@TZhB6~IAxtQjBqsYGRuiZ2ms`A%M$=(C zYNc*)FEPi|q-Z$p-oSX{&!Kq6x^?ZyW#}JIIooZmTep_VhKzb6;S3>^>$dt;QJB!ZZ8g4HYJvN(}k0iF2g87fLF94U}QyLNoR*cX8(2tRH}het;6 zb1zOH>&7(0DFtlS@RIQ_7Kpp}jJn!Cd76kDnmB~z?Wit^c$ns4K98%C;4otN@_dnJ z7mp#~i%4b#l`Xa8$8&SuZ!;<)PUK0VjD%tergFS#g_HTn{R{AWRN4O?3GT;HXX;FT zMmr^grx;DLtv_OQ?eeBmz$%JoL>h(E{oP>BGOU{Vy6nZG`s~flJLQ`tMN>x=I|FIe z4Aur~zt%E;YFm~*HEu5>Bn>S`eGDGyShzV>6U%u#P%edTTh>M6*Yym+Ol(W>+CD)R zlS_-5Fjr6WI}Cx$_SK=ucSLF(VI_Mp+V*rB1;H$drZnA;@X#b}@x-6Hqn>@Yx=Oze;w1kg70F(g{(D89KoCx zT~6tAxP+!x@B=qv*weqGHD3g8RucsH(zS7k4&c3C87u-SoD!6P2+Sx0u9#&JqX{4> zzSLl)5N7e?GoS}b3IRPZqpUYrrves@>GbOFUjeV_ISS+dDjHf>ePQyg+9qyMD75eM zBLbrvNc@C=CLdY$rLgcjr0TdB$gV_-!u?~|%y}}XvIrOg-9zxWY|cF2k_N1Mhy+)T zOXr1WkdwwZ@382!tA%*&_EpPJ$Psb;L=Z4LpZZYO&Z3Th z{zVf$U7Zo)u2lTR^||ja@8we-5_rPe;T;F#2(;Sb1iz-~!R&Bj5@Z54Y(bXk$E;1k zOxtFcBRpu;5$9bN3_TouzC;TeF#v?Hepue_|AD9ZwD|kbx5%p)UY~kwR`p&Al%PbS z_YO{Ge>$jzvsw)4{vE#F$}L&_GRiRiMeg_5JOT9v3(`iwba1j+LgraqbUS+(Zoc7= zy=N^vsy9q zjmh`A60O1;a(OwT#bDrANV`-H`;8))(hDUPr->m`moawhpopN^*YUU@RQ{7!u^&BBf+$w5%Zq#VJHYMBPE#uDXb6;zwS`Xsv+_o&vOp6<<_VB zPT{2~rR`_Qlf-XrX_Orc?DrXXJ+Qk{WjS4a&d{LQ?h3aNRSuWk>?WGGDJs5(G+r<t#F&&YFOS~+Nr69Z8DSHmjduT>PQneMuz;K~yV%p#NqZtPM-^Rc1 zmyN9AyFgz)!JbJ=TTtyj2p^YxmkbF;s_7!r?TB2sd9O1R`<{i7hsSa49rw$Zp;Q#b z{L5hAc(Sfqt((>7uKe#4o(G}rL6>h{KJZm|AMmL+o5=k-gujeD_<{B4$&;mf<>09! zup-XvbrD+qpmMC9TQtK=!)VJWC!WEDeS7cLRXkfx%1WnwWPGvK1=nD|WAp2A!obtv zOD^aBcx=Ai$=Z9j{54rw1jStOs0`OKwJFyl0=yr$$LXIS+SVL(uoO^@;*zAFTPh(AT>@? zagQ7*FC_}OUU_r0rlHu4a+{CFJf2cM*z~7Rk1^bo(Zi)LM_X+(r2*TOq(>_2N{h&Q z6TbP)_~~I#a_Rhi?&s(qoMoV0A%)`1G#7fEE$zLs9WSS)^{qU~kK}5Lg*#f@eS6Q^ zkrIp6(i(2@1S%>M^WjS7S|XjZSe^JU566=GCI+f^40?iwB~`1i0Ye-Pj6lNenVIKl zU{|P&x&F!Y{_1>px%b}ZWu{d=6H>!X8u@&6IZB8VZTo@_=Qo-j;>~s@ei3Da(Y& zpg&z~k0Jxn-ESMg45AL&Gp<*&$sj5Qe@-;FHJs zo9ozJ94|&iWu*lvJaW0czL9dG^s+-c_k;WQ@Td>>Eo#(%@cZ1urX)x>KBlIb?Rm}+ zn3GxuZT|hif!veE>#P$xqBRk_9-~T(Mu3C}uhRbWJDD~Q{vCU5*3~!-RWj5>kbheh zYpg;`$my(GIln={EMsBfwn^s7M=Tqiz8xsW6*C>FSO&ak(EcpWQSn9X-HXk{JH2T1 zvbQLTt%tIpOPRD8VrDY^#E%A$@ZC(#=es7Qk4?k+wEyk}a7Bnkla85OTZm|DXZ5md z`VOt70Ge12wcx=~gTwxoIci1h6d}!XqbC|Eemtqcsgd~axKnRCOPl4`YDk@nyipWuW{p$Y#P^~5*x=DMhULDdr-XM8k=lwU z*`H+#yn5*hI~WVup?Q#k@ZB=@no+4Mh&LI3 zKnVqO8B8X7p`Dx1QO&mJw#LDk3(3}Ou0EbVvov^IRLlhk@ljpwb|bmW)4pBH%0<0y`{y1S?i6y z`@GhV=i(~{aa_~3n&@E(g8`!(8)wu2j%Z>G))d*l>#0e?>)U1cC4hQi$QeLCFdXbM z$3Y2sU?ez~;`lj%2Mh*xWgKemJ%S;P<;`@&7&6F@B((@9>r?m6E?X zZfreoG=txqzU~9ul0j@QNz5+?TB_7?5@kj$1P}=aQ^Q&N%M&Wd+3Q;m5~!mq#8=&U zY&gQ#{3d~LZ)@|T*&;mrw4HiZeGKNI)Ro88eSFjR8asnc#(=x|GGPycrE8ge%gRP0 zy4{2IL&#x!d$aktLWP6d8X|<)MFo?2JhWRhP&;*ZVYSQ-Sw*bw7A>5slCq1T_3F6m%WR&%^t^IB69}s+l_q#_wd3EjuG z3{~^f;wo>+?)2}R_?|ScAjw(zlIAKe?3l+XH2nAmvvlUk9==};g$Xn6Vm%>^Q<4#pA{@T6uv=jpTt(AK9y9UzVQu3r_V!eqN zo{#G!P1rUo4~QA;rFwA8xpGK-vKc1*iJ2nFiPU4+?EW8SZRUua1F{=`OpSuL>dXcm zJt3W5vSG%c(c0Y~RB5Lxl}rs5(!KT_r$+ACiiGZ2FY(Z$1x=dR`PNJcAX@NU4h`*+ zFsQD4p2^_-BwcGiXPmOE^Fod-+#Cig?JyKvd@>2%7ec9527L{M-IKYS9O7lrZAD3O zJ)Rk?6Ifo9uauTh(K)>cYsX(Dd&q}?sIcov7MHJPI*LTR9FC_)nkrKaJrI9tw%gKl zPD3U{&ASF+i&*21kYz?YHLY=kQLlC4K5DW`0jfWVjfKbU68AZYS1&{Q z)!sZ*2je#lj=APIO21VFSJ0n4-sOHL7to{hbl16g7A9fxww((ZRUO$t8g_#6Ycr5L zIp14`5)%-5e4@!LFOyvHwnE;O`gp_`kX-1UvU*V?5_(#X^DIkxJ2Sk5r>&EAcDMF| zTZ7N*Mx++?KiE2}s4ByE+Y<{|bhGG?Mx~K%5u~N1L%KT`9V#i^A>AM)-7TF;cXu!9 zeC%=df1G{BzTkojE`asD&-={zoAdLHRxeh^IDd#C!E2`LT1j-IQC#E$%dB7UJhcAu zJQ9D~KTY+Tz+aZ<`zxkGuThyQ>PLp3E4ft5e!&;uALY7vm|qDczkVOe{`uTs~klTEAkBN>fj~0ltaAO)1&sD2fb1t<10#1XKP)bx0xY_ad zjdO~(LXqEX!7xum(T6lx2gJ(S%63NP2~cgwxtl2X1R2+T^EsJuE{pHp+?;ho*gX%O z+6w&JHf!%eIqwmqi?QR)J6!Dg=pLXCNKBjW1^k*GC&9m{xqTl_>9x4M!)MjKh>io< zd|%CHWa=>m^cikwBsA(z6)HVfh_Ae&cb2^JG9{KRs3xJh$~pUpQ_5+!_tfZe%^2~8DZdgbZvGaEGTz>qlPu+Xw?zN!H7&feiHX7Fp zb=$Y4{T=OLDT%x#TfS0Aj7M&w`rQ`%P&=*iBPyIhwFB8=SECe{@?yFtS69KzeFx-f z$be38>?60|C!OO(9}w5Ge!ij#+}fs}F^9!`<0EB!!Xwg!S@u?Vc|MEH3^AcH#v z+vRAEc&xY6Z}@xkk-BiM_oM7R$U zzeTl_+yuMGfPirY7r*%o9SH*oYPpC>vXHW~m+##0o^)7N4;VG*uA~VY-rKuR&M$gw zCU4J#<|mnYVDq3xO9oua<#&35ADLf<4&Vf`9$2!`zwFqSFGDBQE>`ws=)Cxuvs3!A z^?ENuXQ{2`+ly`fWcUB?*!I%`__Yf~AM}FN@w$2sG*>64`jr9U$)(|P>%FGEo_Uqa zQwcrzrX^$@>AP6@_n8jDoSBYQ>ff%h;IJDv?VJh`g~2rMgB^^mHW4lBrTx36{%E1#+1h!H2WE} zLt>rne&w}+)kFe~Ow6tHa)bQ(vETpZrD^fKAnJET_xkp6_|02(92X9{5ro9U+BjGw-*VcBCQ>=@RT2*#JV9;{gUko|D-K!}gBkUz~GAdWC z!!yW--7=nSJieZ@x5}p$yF2}+>J!y5xQ%(hG5;}!b0%(4$5K=-^zy;8-ujtlzRsXd zS-V*DEnU8PGN^IVw@Izoq1s#GtH&Lh8LKvFZd!##2sf6K*$6`9!Hn@h{Qk$$0nj#n zu>6r{Ae7Sx_Qv%)f~S*sq%2`aFF!}H^!tvWexNe~r?}d{LVRxtCE9TLSAy02-2C-E zBJ^T&pidxffjvR`PPLhtn2S<=k1c=g*v^i<$es6UT|TfhiT1XHEW+0QOyVxbtM^=p zi0bFHQl_x9|B{q2k;Pg)4+Rwc)$~%8hHL2Db259%31sQrf#C5%qi}Rqk;`P#htY4q zhaYGr$gX_6%&+b7LYrsV74$y8m1*zol90sFP=UHnsZZl&v+>`}7Cm!yIoab>kbj5U z@m{im3uqI^wmZG^mFx%al6}mX=QqOGO#?(PWLl4p0oe=GE}$!=H;{>N(Yac+(2cQxPf`-TFH7NnovCnE>g^ZY*+T~M(YAj5u7+M~)0Tc!S~n~O3I z3|qKF=XffG7t&QT85ta!Y)hEZSm}kkhorEBaU!vbMv$X^qXPV z@aAmYx{M&^qZrWzMED2-eNb>ml|fxrZt|pl;HL$*V$3~U_I%_Rxzq5SoZZnhjDd_q z>Q43DFP`fIrmRU%@o&w{i0Vz4o*$*T&tIG0>JZQvJ(TmC9-_HEz;|nl zbGo22tX|FMpqGi~81qlJhqGqV8s4&!&pU_an3x>ovNEMoAe1~OkEaYT2RkfBDM;tQ z1x6}pYp$v3%u)qD*HeZ`NI8o+2WO%#QqEhm5WGpHIH%DqB5~Rt>~IA-mU(pBvX7?! z{*uH*Mnq*9i|65CY<9c~ASxmw#S2*7Qfw=TpxWw<%A4Q{R#45AvoYQY#vrehK27tt zh6b$KAta+Br1M#!E`H$SY7%?Q5Z3f6(r5;CWn>;|x48DhPDn4`etsu9ZLJcJCUvu7 z9_+Pr_LKwBY3;v<9UU}aXyARl)4BObUitnnRklX=uLrU_K(@qij2m{ECBJO}QADEd zQgB(+YqKCt=lN!vGc=sG6+XJ`9;Om6wbMEX$!Ap|ocx7Tj6*j7jhE?RW|Dzz=y#Lh z?46nw5)rZjs10FrH*3g*&A*6x1e>e@knq{D!r88-bp4 ze4u&!^YzthSldF?|({^Li&^UcbHa4rszp^fQyOT>Zu7>eFlNeCt>rdD|#@$Oi z7z|U8jPFc`i8J{QeO5a4b z*wFUdgF&RJk2Tb~!LCD2*Z*VH=;<zA_JlU?7Sj_xH2H;R8Kg4ZXe!WXnm3&a8XUUHBAqSB|=Gih_HCrKcf`*A=_0)9H5 zAF|%JhN@Y|iJ4zE6L`vv^87AW-k9wXq{iUAyf^zC(g?wR{_hdBvhUGpA{M0+EsI)L@n`C0Mx7v_=)hhqCf+A}lQWNnNybv@o z+hEhdRu>_J{1eADVfaQ09qxcUUtfpIX_EPBchRoKP2&q((z$Y^JEtwfQwR8_emt;z zlpLxf*3ohWp=9iW+ewgcSd_ySp>pE3zJUAXCW%|1Y)H2KK_4bR7C(%VzgTQkYSjlRuguYd> z;Y`aO(aU58syv^5O{y}xc=tORtC#>>X8X5Dq`4T6Sl&_|b%rHAgen}e|&by8J5(tCCupSz~59@5Vk zJDDt$qQhbroJVB0NEjI0=kH!?&>B8mN#XZJps2#+CgLY!o1GFP_9h5-5#LCK?U5Of z7bNi2^3=D2tmh;gV28cKLW!d5f#!hA&TEe4!d?NMDo_YIjrM2LMV)vPdQ*Iqhux@R zEkG{){dOz3`iz-mDU4()-*nCbYIu=UR`(9H*I>V#Ofg-guOx2e>~9$ z^=Pm*3~cFW^&^MN4m&3;pkl8_|E9%8h1!Dd(Fn=p;YS93%$AxoFXYli-YC8B#G?No}Il-L&GV zC^wl~eazVm&A!GY{n&cz=B^cBHMfJguL1f_dyn@@3HNsO>3iAEC&;>CTj*{bd-$3_ zWC*K$`VD_q706!i{nw}fNs2objn@c-r+#Rl{Ns5awb!8xyVe%oau3N6QtI~~6@OSK z9))sgQL&`?D)CzDV0I4Ff@_uo_VC*Uyjil@2o>9Z0Gd%3YyYNf#Hi@MC+!Zm^Xsw1`|%Vn-j7+WYwO@RyMFxq!v= zPrP`hxTH9JK4*M0MSb)uvZOFh!t-wRVBW$ggTc|n%XR({b|)2xU9`w}5-g)uKI(X2 z$@lJC9HtcYJw@QSEH%>uVbhR`av$~){;B-&7-;qfnS~!HZ;<)!&o^s!cNp?_JZ80G zptDggFlaC#HQnwiWO~;ONYlE4=J6fUm{yt_GCTfld3XF}>|zizn6%x_OTf5$XQvmL z_VGB+D2FXa^(5=ovu8f%g19gL{%iWjG4gfxM(kDle)wL$PIFn7#Ass)$w{`N|D8vzX9l_dHFA66|ISwb5Bh@y;`6*LqyObe)cBG`fbsjt%alj1tG?MM#)>oU9y+MU$ZqDv^JYBb| zO*tS|i!XqGLpl%e-h(bcfO?FgE8`zf01|1M!t4v zX%GF=X|783yIjxrE=7F)#h2HZ4n1Cr_Fzj>p<-nU6}@~F0IWq9&}zFoJD2w&gR7MY z8x;oCWY=p#%kDC-Wl{u~1&@#aIe~UOT+I7xWvO>;_BGL(ve~Y1m#7GemJ+N8`je=F zJ63WbXZ%~(BcC^y<;6)vr_J-uj2?V$sYFcZ0BPtmTHZ-ctGUOs&NDGO%x&+UOHd>3 zHO~@PQ-D647&Jx56wkxunP1{)SUjK^>2gM0E($O=0qc`6QuhQGit0b=obW=Dm2q zOZsObk2L-3??idldZkSYo-whr-s@ZBMcJ$>)-+*G*#4&0mE74JyfAlM=AbXvt7eIR zs}b8-+k2p~&Np%L0asOoLk}{}%MfDx+?11DX*YOf6(6LXAlDCuv3_pFTpNmOIgLA) zNfO`}m~+`Ye0ne!5j@@BwmszFzXRUSnkGJ>-ySW!D*CG#ev3J~vU(_$Xhmzh0l6t9 zkyRr(!|gehQ&P_&jX_krUY=xa+S5f*%0Uga8+K*J``M{}$@{iw_SOE7Xks9Du}arW zT7%&zUIO|LgrUrKA3_b=#V)(Hcu3>L%jzJZ^z;1(P2% z*V%SDTvtz-H&(>=&0kyX|Hi)8bi`(u+eV!-m%JUWjA7fdKlI4vCg2{X>G&J7x_{{Wu|XYHhn;HOdp%%{;xj4t%FT>PR+_bD#D1f+S)#lCl5A>s)#H6#rkA z78o<3Afyj?yTom9N7ECMKSrleorBskZ_k@XTT!ynzxcR}-J>iaoa~l|`L}tQe!Ge* z2yBzQe;j6Q^i$l!V!aKlw8YowObB;hZNM$ZwL6!TIl{pqmkY>H6ZJ*r#Wt5oW96qi zyFqBr(?5ihYj41!>8aE0|B=wqR830+_z_+jvKAd{!u}G#evl3#v zPhW1eSO@_P6{24lNBO|$cua+CF>+-H*{vT6^_(VVFjBKCbB^e57F&>hvHxx`+-3;H zP7tcumO2< zer`PBRHCj8Y6RwoC6s4QMEFgO!#w%Du5SGu*xu0lHb5@T2A7n`2g zqYNIM>GFqTg)v>Elr#GB@eneRx`%q3sVkgg6#Uz0#Me&y?Vlx7{HS($%Xr$U%lzg( zK9m_fw6{Cdfcwo(DBpx6z^5 zD?OIXI!qsPB7+~ZnRg7PgVX|fn*s5)(odO+Ej`@2XqczC>znb$=$O2~hE|EtJGkD% zkG&nGD~AIDDHTet%W2SZQIj9zZfa z1+C#X71JzV6jpY(SRNQ2Mzk$rt3GKDrS6)gaL8G?mF8bGWmlRlI;nrpCUYP`F65!e z=Gho;o5ehK}{a{wrB1ux2*0`Euq7v)X;MS0t2%YRYc z&R&$a7PDB17v&9@3py`zp+VtAc^mPqNzr~$-YWhV<*lJ;ju>s{HC$&74ZCTlyvOxA zKLMLEh;&Y+@l(X@4x0z*X#^LOdJ!>qNwM}HDV)iErGMx%L~Ob}*(XmXVTQfib)O(s zs|lIqf|A>pBy013zpd7LJ`8d}6->ptyJF_ev4pS#f+rQEXBI*_a~xhxO_B6Q7n<~D zr%#PvCf@3PT7N0b?z6NQrZ+}}W# zksrvp^f-BU1x-g~`5d=no377r6w|pZaoEbpMQw4rta)eV_flgZ#jIu!K5cz5!z+;eaj zD%m+9-rAhI(M%>tcrYt%kOQ3Pj>+j^Jd#1FGufxup0whVsgB0~3dcg4W>~m<!n&N>2#35w%r)HqLrJCh*~je^~I6JY-(kFV@zaxa#KWE<#SWgv!`Ua-7C zOpIMbz@<-`QO4=F3vB(+YaMVG_;j|u7x||~Qz2L7mY9X36AWU#6`Wp$kX@aKqy&71 zMd?>K(G)4yu_N|w?tg607!bZd6^O=rFF#sQ08$vA8^3zvYILM9uo5d!6M1B1 z*iF$_nQoY(Ye7*TI|D4fb!^Fof4l-6)n-c{4?e{;aT+$^Q+(yE;O`jMNm|CFaTV}$ z8Ek{EDe!?Jo5(`Bhr{i%7Qdmo(Sx0*Pn>f^1KU9SD%2=TKM;4sR>Vi<)%gX!&XH$V zww6C+oU1ze#H#r1;Ht+J;~ii+=!+dAI53g`b95l=Hq&f<1^pxyU8dbHU&xUJGqp zn!7R~UZxw4fZiLNho#mlVgc0B%mAi4;wE)51IY!ss?&~7B}pjbLg&}zqg<>myak>S zee?4<&NDO0Vk@P$U5E%!*8mgO_lwMGLXenn&VUZUKMw}JRv*(M;588<2;KE$H`02B z^I#TuMmnP1G`Bsj~<)Rl#f@%TmG_beg16;)(QSYJaX)LkC5y3iUVG~*9PTGqSCL39{`i& z&?5s+6k@rkO3iCbI@WN;b>WTr3w-Xqhguv49J|CNz=0adLVpG@wyWMi5~t#%m^Y90 zApOZl;M(0*xE#|o{~ZYC{F7DsYat9^u73RhR{jzf{7;yS*D7O* zn(5rSFtg<=6Soisd}_n$@%HHWPTc~pAxl^L=Av|t#goKgmE66ioxaIw*mZ)#mlbiQ z(9{0AQpnovUwdcwtF!Y^8e!PQtJ<>~hVG+#_XOihJJfUAPwgCZXJTisZNYLRVCQR4Yn&u8~L3Ip#eG60gIT@fT=(FAY<;q+L?c06cz%I`V=aEsp zJGA%v8VQra9^`i;M@?rx1sNDu3S3F6g6KN?IN+Om(FsETC6t^KH%3*A)+D~2cFKyV zc%xbY{F>jbRucOTvri5Q!9{wrC7Ta}lI{L&;kcKrIj$q~h@w<%x6Tgk`qP*F!Tb&z z*N;p!IODvsmd01WT2P4}SkQj8i&4SKih=T>#itjm(5`|N!(A~`n5v6b--%+c7ex|w zy$W^mXuTJEPA3MmlQj2|j-_3(zoX@;;sBFqLU@S@#t3dNcp%F!XusJH6)O}Zm!4Zt z-yJ7Y{aW$#*0%(^@t3I)ivHx_w3vSZ#P`G2ND9Ry^=*3BN*1KcPwZBJd}(Kg&0wU% z?D|hiw(CZy?w@480&zo!uNptTryX@ODy^NQlt>lz)oa2#Bt)!G6UXv>dZjPSylSFZ5`TFUg7e1-Jnwu)`3!+t1H{kd7oCwyj{e7I>$D2)%bd z0g&S5Dw7lV`MPJrTD)=MpHGGN--uS&K3F^t)sdZ&Kq};3gvtM8T^$c!8s* z#Oq$nwoht9>|B_`zCQVfjN;}8#md$B2#A}69Al1$L2YkwF0V9$+AO)~!ro+<6L$@Q z$w#xGW-+3C4*WFu>H4iij1dj)!{O}ILS#Tq;}ma4{zb!#XR->F>pHO;_>`Qt71Hc? z{CvW#{4Dgl(Cah(Rj1!V6rjnAio#sdJK*Zu-n>PE#nQTLk@gakQ-s;IH|1ESl_=-E zXZ`JVM=4LyuCgTBbQZtydzXj;VgQz6%>}tXbN=S1d+!1LcWLQ{0!w5`q5?a}2gkgI zf7*P4tsbj-L{+i!ci|BL_~1g5oZi5rq#5l04F_r5Xbtn zS>bcp!Ox`i1|rJ1+Mj&1>e_BcB-P;UIm zt@rq(Ccm8C-l>8b9N-6&4tS53w@v7&{%Npjl9W7Y{a6Z?R-%YL?))U!XF8l5Lh-d# zxRv-kqs|8&fdkt_7bpL7Ci8t?U>neg>)-O%86m*pIrrHi|8ETh-honl*#!In|94(+ zY{>VnF`ffXVlJfOBpi1cDk5pUQ~%$7A8JV6+hj@Es#5v$m%ZhLtH>o z7^7~JuR0h1$&J+%E~k+wg|IUoQMn$;Ueors@AQ7bX!%3zo8B9vw#m&-sJjAK)_u{D zPyk<+l-fAIr4%#JgqXW*DxmY)i@vYoLI_YYcX3fb!=@7L_XcNBwCT2cZBSBK*H_uq z{ZekWBJp_OhC`0bq&#|{$*wGL+Fsx#x>WdfoVE0?wnqx;I8XD=NP_V}$O~0z9MuKV zrnWAnT;&xN9gZG}a(Ni&oQ{$XbSuufJj*`Y+SN!b`rs`f=G(-29KNvD39v9j!kf)5yxtnW3S1B9DjWgo!97JYMvRabZ>-_djx z)KbSRW}SZ-LslP3SnEv`yusb|1TQ(`bqv5m@5tdB)GEw~$6L8>vk!imK_hG{5mr^& zcuy9C63E_3rmRb4d*mQIJeu-!O_<|^<{J)qtOnoqj{~RZ)H~|}j2Jd^tcNFDWN&PW znFaU0`WI_k?55+$`^M;RQ{-;#6_yA!ofsRKrU<$gh{O@t)^NNW1V|$}!6 z5Au6c;>`t-x65QamMp%!-^SAzGcK>sK~Bz!DSxs$_4jixk}A)K6@SXpL0*zH6)yqn zie^nxa~dRaq|5S0Rn>unO1Q;VC_d3xYKi8O{O1};Eej?APLZ{P;>m(A-rMwVx?ap* z27jOjo_dBm{b4*)YKlmi=&hpl%Sw6JaR^T9ikJIQj{y4++w+F;4<>BZS{KW=Oxd5} z)$@}xNpM)JM1vmT!*qeO?LeGpO=q_KLPZ^o{nMM>ygP&?-*y2@mM)yrgJ8wd%LQh= zzD?!IPyDB+PNojdL%#`Sxv$=RMiXVaGtn0j`F*2H*RQN9wxU5{E%DU#mwMF`EKl61 zrI&adRj^Qr07FIz>0WJ{2x^OuUY{iU9>dmIzlrg1>4-zV=!{%p&{&ByovK2$)bzn! z=rsCW3}8CPT(Io5`vE*1S$fXOT;)n2SJNW05O(O=qxxt$?3@#Hc_p`xB-vNHvuP6g zCrBFm5;1FnUqTuMHpvyewjsc2)95g3MjrQ%tzUItft`W>Z)jMtzE43qBb@g@)=?*f z=EQR95c~3oD>=RCacvgBp*%*-4(%TzAK?9HH%`6;8lx)n5L{9`Xfje~-;6JF9om#w z$mn<>OKp55cg7#*!g0AR*RLcHJ(+J?!#SKMASwHkiJpZ@O)j4;#Q)5q_!qbZ2DaUt z2!)E92m~ZW3#N-(xF7v`jxVdN&k|YVVkO^@RSs$ci?gNHedI59PeT*G&1~n{xei^0 zD#7tYG9#p*Qet`(oc+P%;m8;9LZ5Yxr9J|r;%YfQOX-^~Kp4nIXqqhO5h;8ZtHUGS zA-7YaKaiaknUih*?U$(7BLiW;cIZlz!sN&YDEZsng-VVu1F`>Dvh4v{IES{WaH^dd z^Mn_%9YxBQLABn=h*EIB&*$%%(l&a^ul%*#AaBKu)(eu^%K37#5vzWnkilwFuJh#z z`V+WtlLDn+1A$9J_YVkq{ZIG2mk0Csi@Lfr z+^;Y`D3BDY6D4fy9HFPV*$3HryIbl_2@PbuW(yiRuCT))#sfKP0+z876$f=G$r$;| z$sLCr-Gk((SY@-J{X3%L?cLuWvp|kD9E5yFT`SZw26v1d=(A8M#W%^L)_jE59X zeR_>ivE0-dV}GM@MC03XD~8qFjCB+grJ(Kj@8CDP+w)oX_|p#2?l}&GRlgYxcvW|Z z=cA90k@Ag5y4ypU<%9rZT)6#UpwwTfv!RiLDu)rY8p*b4LwIQ8!duvmHyl*@vOmOK z{}_DM*4g`=%c0uR+rioO!~6vgt8Y50B+M1gT?h*`+w=W*L#J z{%ESHz#DMHDqd~#zazSQYfu7GW7SCw&<`X;U8Yk`(ziKU3uA zEWF2}Qa51Wo%lmC*?BE|KeO`_(bVK6A}R7#uxj#^5W+h`;sh26yTVHSQW0}htrs|_ z;cS0Ir^;;VUAlofx_gcJAjnjX0MV&J*j4c=5HtU2GqV|M?cYJe$>}hKm3jpn()spU zZ>rq5a6*Tx3emAiU#CiK}Tskp=#is(2%h1 z_V7gU4)mFwyy^{c%dDKo{GP(Gjkb)13wJWotY-?Lo~lYGy}zUj%i_?cnx`FBgMi-B zS;u#qx4Nwo`~w-ZWPg%Hgk{GY%<|EcODt$SqEEXFRL$9$luROZbuZyJ`+Drv3Xus0 z;>h$SPrAx_hhHs)$$DG9!4Z?+2qDi#J#Te6YMKr7qpz@ed1xG8Iy4-CRs}1O|2p`! zTHg_{enK|-bHfhpvH*RSz&RJ^)p*c9qs+=utGvPgGMb3qF6b9HBeyLX@T!+&HUDJs zHPN5YpZP=UjtsubzrKuOcm%MesY;ZGDvS?SQ!jR!lgjOkem%d(=!F0+CL=96fD%$J zI1@~T(?xsg+T;IuaR(HuUZNjtq#Ovl|6H$t>`LW(OUwyN>T;mhgxnZo-;AcW7UgXOWKwh~#Tp^~$ z>d2DZso~l*Z@HOoy}c5w^Uj;Z0aWNbo-Ye!wmeL_LWKe>{Ahkp526+e`Sq~Hy$wyH z<%l8*MND|;#v0em*_1ae94R)|kziPpuPY*BzbaA-58>roZBAZ*O(oH9^$8klH#y*Jn1JSlJawLZ*ew zX766~cU3yn5p{e91r~d7wBIX;Ti-qmXPJ>T{KC=Sr3^~Aw9>~Uc8!&nI6a}bWrenkau*n7psP&5({Ub-+$`S8m4nK~W{xgoW0i_cPBd*=cF+7* z)!R`y*=zy7GVV+C_3J&Z&D=sY&S}189*d%_yxv{NQ*d<=82!ek)&Xf&(84XwDmwml z$`zT}xrZFsEO=$vlZ6@NrMT^vGzsc;-Ozz)0e5_DM`k79THmJrqEV=ee`I@6kY5UX ztczGpfxtjV0~7VdY(o*F<;zn*`XVP|&g9Djr7vOzIwzecEl)2^GeJPu|D{XlKTR`a zb)#<&cmpInFS>PH!qKrXKkWQrcT3Ml(g9tId~lC9F;^;~Fs(5kLMtbIhqUwl8^1oW zMAPQ%!QL@E*=z;SoUYR`FlfTxjtfP8^{_q^jzv_igEg|xZG0O~WV1_&oym?DIQ}si zA)sGRr&b!}_ZYj!gG@zS8_}2d{fTa(4PUg9EAYAHj(>p`EJE)}lis6OS@rRl0S2A> zd#b>^R_gT`IlBWOr`5=(4ZcLqv2qd-@sOw^{#wgs@QWJ&XHqtln*%4xL$eK?Sl!%R zrFQzoZk_u?m0QG&EL56B`f2&R4>I+@qH~)YNxK#oE1gi~W)&V0tUjb)?a^et_m_n) zN#}fd!1nXS$}Lokn7pf%801tY-1t#Li{xZe;AVlWJqP&Ll@w(oFk4cmKjIuq)^@^zJhP|N*4r@a^QI-C`Qdw(cxs35I=@}WFYV!rA z(y4HWLG}}#q2zXw6Yy>T3P1oRrvJ*RSzjguG*ibU06@OWu&b*hym_!TYNk55b^>f_ zhazBX(@P?pb}y4AY-xDTz5@?rx3VE87=Z~2dHPq5KliPc4&?}EI#w47ZEgLm1S8K9 zaHm=HzWrLmD3+*ejlX6NYW=g(9Tx3#($v`KL^i6|6bQ55PWq~h01z|&ZDY8b6oOX< zu5k}>x8g~&BaMsA*pywhp5eEtz{D9Ge_1l(oX_oro2sr|qWb9IXPY|buj{R*;hesV z{K=}@G*;N?;CD18M;|@J`hea5Rm}^vSp#rwWHb)Ba{7SZ0s_QXi_T6T-w=vW!t3(^L7kEFPfy2_0Adh0q;4M| z%~B3$hn7*fCrX|^1y0wKJQnlWlm^#l3SBF=|9I9=^7R`AopiVt4W& zyyvYH1~2eR9WUT+s8wXd<2G7hO*&5TAYIa*R((8R0rQ$~t(dOmR8ljsWAw9CEm#{T zy?4NXpO}$r7vKlZUoWr5&aY<~x4zjvgGjfJf-@(h(cq0Zlk)_0_tka&8T4HFVr+El z)hs;IbBSD{1ip7ye*wRgn*0vFPvNAOol#G##)kKB$&+P>9GG0pjyuZ$s#9@!dw-88 zm>C+<*3&qUu=GLtQ6$MPDGd#FeS2j-005=k&2rl%)J{ui_+MpC`QR%zWnkMgfutS@@Am4j;0bp zDGa_P`-|j9n@jda{EFZb_kwiO4E|t=6%2@KA${XH86)-eLVQ$3M@+#py2~sgc zKR%9_}?OLxBgX3*pc6{TPA;(diPc5hd zJbr@ve@?1_Lb23#&HrV{CCDX(r4j-I_pNpg61$Z$5hP&ZReAreFQ?Hg!Dujb@@4U) zQ!2sJAdZORI`3?McQ5DEgb}OWwS1N77?wU9SJ|9B*z1k7M|RFq95saKid1sV2Wh8fPtE?;B>5o#Np^VGWjP7 zbP1qO?C)7)L|nC5!(2bK@~u6KDjY)^V7 zxB66<=spORx%;^&UUzZ}8Xhb^Pw%+8?^83$SJ}G+bR1R$`9`vS|2PNy$v)QG%YI?m z)j8|`IbQ2a=yKG8x=~o{FF5l3LC}|4K!PzCbqdnmDEIH?#TEGLXn2}T;_y77O0Bs} z*QURV14HvA5kP$+U!)KcUc`xt@HYs1?w(tk;GkvVBYInPQ+UrAP2}aQ&mCJ#S zst_rDKjqfYTUNm#S zVq6^KMObVIIs$x4#V!Acwy{|)`#VtQgdPeO$8^ahIGOB21oHBV= zMqbmcEg~QIXCPCDblK|tOe3Gspgb6uk%>-Cu$=uty@Iy0XEW92&_mh$t6=CgBtOR& z$c+)^iM7h)ZXLYqObJiou2DT9o6`4U1~=jrDyHb>3tywcF98-vujd4co81E1w-%qZ zp{ukP#2le@kfOGgCWNar9Z1~XFW6biW2^C4c-dCZ?AHuD|_R(y^4J{QNe z9UTXS87&+`VXTAvF9fyf3y&}rm04!!5+?LBg3j}H*kNuKISx)`5^rgBXjAX|7~A|Wm6V)Y0Aj4&rc3- z8Yst%BUZkMf7!>~#D*H2U!N|RABYoKK9=+o;~$;e8am>$MYkg&qJ)u_-;DF-%991= zpz7_}og*z#XRDyn&p-B;D@(+1kIuUY9v%i)oE*>}ON)BhlF>~u40@c$CRVQ>88D=Y zA5KJ*I37y*h*doSUAIWTf$nD56qHk_q?_ONUhKBxhwbf__Qq33)AFo=rc=4X`boFl zbfe!ukKM1VPkyOXmOjgkcVLaTe=(Ptm;Vy%%0&%G_Pci8Z}A8=#OaeWbablG0 zib{!&49DX}e^fB5F!mQ}tE=L!|-XkYqjodWmjcEhV zJVhrZy*O6&g|)G#L^m+O1E6_mj<(E6!Hcu@D)1EhXC#nWgS`VF4&D{QHD@%n|7gW% z$x>=mBYW$a91X9GAfAi9iMDt&$56uLvEmDR_}0%Yys>6(H=me;A)+L=FBk3O7zWY%ydw~x6 zX?o|^PqK5Rw~^<8QH2p%t%8ajp$R2p$n-&WkED{JZ;C%X zLw_u$CGc_B9z5w%$|eq1Mty)eb+l!V;h^dj|D&0FTqZ>B`q`POi-N`CYelm_GrN{7H5(Gb_m*{m?xR)d zUByx--m~-5_td@E8SfBDw)4*)e0*prP(W>BbB)TQj5Gpe5f7#~0)71559)HOJ$FSE z8Ut&i6rF3y2>e)n?42|01wN7RQ1tCV6omuso?bh9qob&|I7AgrBntkkwb%j^@ob++ zY1oxK&L2n~I6PaLEErw-Xm7p*7RZPS_=X-ggPL(KFP^%+b^YzOZDqF**4PVtosZ}P1)Rj80T zb+-5BCep~OS4ZBSO_GTOqYgWsn@A0^o9v*-!M8m1LH=zMum9CRtz}Y#NV>oEkEL(* zCe+T4lLXAV6swB=c)0W6;0Oo@v9kmi4uQ__tAbb?o5m529fD0(a?tM%c>ub8raAyKdqcn1Efba8%Eq;vQ>tnEF8t9UqsnU`)_yO+AqL zijh5jUo1w-*hQ+Ksi%J|W@nd9DO~O&+0vN0RUvv)?!*ou|B?zA4Ry_1PtzpdF0?*r zU#y}JWQ)I8Mlydze3gHWSTOaA6$+euT(# zGMPrW7_;^?KHbxjdl(z6bF@%JL%>sXgv49tGipt&9(nzo_pv zdD*`rFPd*Ena7Fb!3nM^-k(3?yrg_ks~Vb%-Vn)S$1{;Le39%wV^I2o^~u$w@8 zKPC-snt|YLa53`wux`zP0-adk0pOeVbStheV6M zn!b-;@`1+DNwMz|2ByN_KB^;`@tllf!na~LTyK5?MdYHm6#JO35S-qb#wG32g$-kkg?llC|Xv|Hv5Hx5SqicUh%N7*Z%SZaO*U+AM+ zyDAu*Z;l5lj%{Y+ziIQzhv^mhw}CK~fG)BAM8TM}ttFl4yU$4CDpn5Ha;~Vp3i0!( z>&C(lad(C-(cr;WBfiMoHx@MKuYBjW!!zbrO9CmLH6Fw7%FCL@mb%m@s} zJE8VaD*}4Bw>65@4VX6)3IIn^pLT4*^CcQJ-ZQFVuU=_T>F+hkG@eTSRT$7$*foK; zbDM9M7meOTS2&iA2V}s9J>vv9dOroyT;+H&S~|_NLzwq2ZAFcBGKExxB~`h9ddNN& zk6Dr{-%<(V->v@~WpsH`UJ^dbj9{ju&-Lsi+!1UFvpmi<58F(XzhL9vSIHjx0pcUF zc=CPRstI5l=V>}&$IK`^D5%xkdrhwdT>$b$8^7-b6bU zh016@<$k=Y1X6qNRzg%m*}i5w`FlddRz&4+Itgq)7V^w5EQS3e(3d-0i1;18&zf^73AB zmq%~Zm-2K*c_gSwa{dbsyd_R9XMh9<@J?75$`3&%51q6LuQTBl5}5vr zjV>Z&|8lG%ok#&J8snuet#e*vhMON@zIb8nbNxpDhrRcVYI=LuMFCMzlqyA#qKJrq zfK)-ch;)_S1w?x9C877GQU&R~_YOg+O7EeEBE3Usq1_o>|9#G0``mHQK6{@r?uYeZ zjj@6=$#1^%oo{{KCu*wur<{<>G%KCUA?Kx3hfqYsYW!z^k=gJ@6g?qXvNn`fIP{E4 ztlCv5ms7mohpH~S#}$B4)fWx>Bc%I}sAQPbF(I_{KlojDo?mE76W$Sx^2hehHJ&00 z%n5B;86S3KDRXk(uGBH;zAO$(LnzTXPX%)R{R@dUGS4bf= z4O_+S80g`xLlK@GP6m$FNg-(>D7j+iWE#nP+&@o<7JK-tvhsnSlCyyB1FYeN(FbB+Bw;Y($hR^Xv+J3v99Ry%<@KoOn568auxl-`|QbRAVCPc z(3?J6K`ledz=NvY(raIZ)eq46YGlYN-T>&Q+aoa6o+H4&I~lA)ogf0!Qh#^(X^7qE z=QW?t0D%8d7IAlV!dmQ9Sb5Th|vXGAe0gc$}*X(S23Ngb3Wgj|m zw^lYtJ-5n8K6L9gfNlHNLsg|?U2+WowB&zm|>lB&A3 zWaU^f{)?TrnXyKgV+=ahq&SyNS?@>GRuPK>6Zrx&8Y?EL{)H2uyC=2g>gVM-Eiule zO?`ercHQ~#_6G$TqyD2i0GCvblh$hnKQVJJeI4%lMzU;{RJ$MUEiw7xO`!nXoA$;B z!{x3n^6ip^RX2P715wGLsY|2n=NJW#i-e$KHzw6xw*ymNy}=e36Ch(Lg^%fTm9~$4 z9-B9k(ccKB{_fvM?My)R(LgQPFM)m@?5v>~3@UEXLHU>j79Qf1MW2_a*F zCo9h&a%J2o`D&{3j1w}2C!G+esG^*^c(FQ=`avcm+L?`e`NI)R&(B}kBc^#KzXaSe z9}__J6WMP_8ALsD6D;McuPQ-^4{1|VhR|uZ1@($xV1xehOc}-gM$`^|c!tHB|TIWaJb{ zU-1sPB|~2z&W;=N#5D8_FMLo0DO`)owN2eeE{nf%Rh-{_MBrj8uS}L}GMd7(xLI|o zf^Xai@!-q}G9pCrV0N@s>l2~&N?NXMi%HUO`InVHt^BB}O7nI^>{u&04|-c~>+CAB zS2Fog>Gm~pqj76hCe!Zj?ofY3L1ipC027-I^6*HmoSyIw8SlB#ay`qST}J%!-ThFG zZ630?XKR7W3Up3Z_UGsz0afS4mMKq z@0RDp=DXVZO;#Y4G-}PM8&B4ZkV9YFY_8kb?UyMTzH!(`w6;g{$^^?K7!!T12J%CW7fT+OwWmsZK@-yi#f2qk~M;^w;97Gb*Z-A>}&@efgZWDt(A=ysLGWKLe0 zhAL7{h4kmMhs)h+B37{H%IAD9HxRWp%KHTyOHJYnrav`OD5{-Pjkfw1GiNQ!-ZMin(ro1D@Cjy+DY>rvp^^|K=FdFP~u}^ z67ED{?8Rgsb2y8jrSrPus^J77D*ouUFKp?K0robo$ckBjpf4=U&T^!Tv{D6+=Ja4g z&SHPklt}q!JF!MNl(kT;Jh-zuN>Sa#Wr2Fwwc}PQTS&x-Fq!ObGpn#wDK2C@yr5fl zZ7`L@VSP%1a~ZPEd%NGXEuNM)Hd>wJWN(b+VqW<<3;*H1e=zK@kwht#Z8`k?fv(}r zlPf^McS-a6V?E68bEcCXtdB%aR9c9ynwX%pTAOFM*GGr42H!?=c0oLn*d?khpk}5c zd88JP9;ZER!%Seos%o=s=ueV392prw39DD=v4Sh5iEY08qErxK;X@klZi0K+(lb!8F z5bwAI=cm|Jrl|Q&$qVbby{FMy$eCCc4kY5)f|lawKWlQv+Q+~o4D`dWRb{E@C&Q+E zh>>d5q9wcO2OQ#f-8o8brK?;<+B6m068n>;w(5vW+q_cI`_+>9ZEogjWj_^Pl@90R zCQqOU@V5}7G#FQ|3YMvP7|AO+MMQ0GUulc9$8uJ_y^rR?@J{%3EH!Ks@5d-?OhR^` z|6LEXk+-nu-UVaW#xj+ykb|pm8(3Jy_XBJ|9hiVnvwox@*hxJ5dAny0yexqFJ7N7z z{T=zl!96IicteM*+_u4m^l)CfOt`szYR^ZA-R?Tu9>) z<_BQ42&phxyhFptHNXvaRscJ2qkVS`T-#C-k*(mwJK;cdJ z@~0O8{-D+K4HhAavMpaApdbF+Aas@*eC%ckc!6&O{>M6o=Y}urP!K=0*kuW9)F_k) ztvGZscz?7h0i00HIo|pP@HgNcMz?Vx)WjcXfy3z{Z|tJSNG*Pz+@ep2%MTv#W*rWP z2jNFa3>TTCK0OXxUoY@xiUwRN5fL&VU-b+BN;<-8K)iZ8_svV>z{5?p`i0?uZw>&$ z{GXN5EPxYeC!Y+}RWaRD8uWwPp+(#dB&Y%;=r>Ycf`{EKhf&Ksb>ZjW_}eCxbApZF zUoVdUyI2gFStJ4u^7mt4Ct{F^UYqub6UO~P@7ci6w_HMj-7Es$d{f8w79Cv(5FCp~ ziv}Ssn20T8cP8Q)1KL<@F;78&EkdVbAqHkOE}c`Y1o1g~@qNz*!w{}^U|6jqfxabh4UmJyx6q3p z8QcVmd9Tx1)k={-ynZj9`qyIgQ)xaJAOMS5!Q!e{24C61mUv}nd|Bz}d@2=%{EG%3)tLxTJWwE_J5{b;*_#&={kBfWHbBdvLfSm+e>;*9VFM55M z=;#8$bI=*JC=udb2OIv#NBqIllMZ}`_t#sBqI56NANWf)4hu^R{ed=I-Pm4)=nwqF zsTa%*bV)-z%nQUSQ};bdV1GTK9M;!*nAUM&&pngx%fjl^eF{EdLax3etFkWp0{IJi z>@L_ZX>C6%r)Rsff3of|B&{{_Ihb@)$Twg+w(Y33hF@t{qbB$HNypw`XR2H3}jp`wv@iqnvKygYB=m7P1x3V@hL_o*J_@RW$Ut) z34PeawD1UjmX&Gwc0S%uH0}4*uNTZW*F9xsJO79M??-y{v;u@Lzc)-7(-UWB1J2Mm) zyhp*UbZ02Z;etYHP5(Xamn(GI0bwcK1~zezw%+oUQ*Q~7m!4jLQi+fq6TR4qS>KQj z%Ndu6wxtdawab*KYhY}$N66#HRbTEt$%AD+dHA{PW1x@7g#P{95TSAS+j))od5c^d zR)(xk`Cmkid5u_OqiWg@B##XyN(nhOKeC@1pU#Zc9$MSRs;{4o2IoRHW0Bz3sgcT|arM+huhPt;XoPeqV^*z+zbv2!?Zvh zJF@*6sK!0QdaH2kWlQ`J54;fdv?KS05vhf4f(jXhhW!Q+kdYu3+dDN}$ocv&Dz?ij z2OAT9G|;W9@jPZR>m=OMP3t7#Y2v)Cwo+13t{GtZ)=2N}55%-6?pk{`eIAQ^HzArL&K*Pu_=}>EBs9k{;-Hz8Uo{aOG^yi9&1a4PpXKcpvc~0yf6)Q^S^HjF;j~AY6U$xFI8G>h+2EzWA5pXaCP>gmn8Xmop%eYknxh8O!J zb7XxKZqh=kh=W`eH`MGj`99AutfFi~ozkxN$>@#L^&Zr!k?M=6=e-|AujD$&<7Z}O z4PN7hE1SUw<`=uxyR#t)Jj49Fs+W{23AKU@d* zMfpx7^DFz6;Uwu`=|=**33ZO!FV^9fDYfNG>o;lzPwnBw9`WUL`t$DQeL~ZZ7#R~@ znV7+gbQHQ`i?a60_5!Pdw)h;UQn}1qiz=MwY+X^Gf+`@Vhy2=yh*|=}4^@Mw;m|{xPLeQ$8@Q~viAeWQP;xZ1M5LR+1GDK4PpwRbTMG>P@%8Pw)F zY7e_h3Ag;My*3WS{XRf@?;8D0dkH!l^|xWuOcI4udV1!Un2nFPS)-xxjh?_UWr_MY zrF9qFj2B$U4y3*Qp6X-_kZZrqzj@HIOeJHoxW`@bVT2#rHNwg!9Q&kx?e6*cuj2s5 zylu|)BF>+kD#G7jifn=(gbiC#rX2?lMON1Hs7(1I51$orO!+WY^(nYpes*qbD~cwD z4{n8xSMwCr(jg58$3TG=OG0Gry6|iH=J%kacpuk#%Ny9WkWy;U4G<3r7rYN@+>*0b zZ&G1{_8eay|E}}ebfZ9K)sg$@ZG4X79rSmA_HN}X=N>WQx|8*4^tJb0t*5KZP+_T#XPcc$POPtwK7tg*|C z?IP_#gG)H@h7NvNpTiQ2?bPo$n8U+Ae~v`%<0L~>CUrPD4t~|N;Ae^-P6vFB?#awM zKX97XHZa(9JadmtL=Dx@AWuI%$b8#SijR=n-9$DrlZo zn-nmhc9^ALwbn_0u~I#CDMGDlm$1<-Yp6*YZ8biB_pOm-U*jRa+zBcn{%8jqcu{B2 zG`p45$G*@G8mp6hREpc66&K8c0xd6{n=i^sKCIBx_j?6w%8ab(L~N6s2<5z;b}G~; zjRDR|(!{h>%g~S%G>t#bIGB4AD#9xj!aNa<@9uW=Pn@+lCBPn@ao8}vJSl-;Pw9O27U$ifxiA9 zv;IrxLEU6zR1JRTyajI~Gj~yDD_6%2+scGw3Xj9Zg2m6o@IVuf_S7Ktds%X?!!|hM z-4=eN&m$JJ-1uslFOs~S9)N(h@9zPvXsv3)ac6l&T(xxJ-0U`kX3kZ1mOWNQ;pbyj z+OP{~d4oHi9K|Jnp|)a*k4eNE@xZ1lJwJQW-Ybj7!abE#Rr9x>G6#7~?M(knuA#I) zrRIAJLDUhD7BFW7Y`JN8RBO0lB-u%u2G3N&p)Dresiz=-r25#^mk9~iZYNiC`nQWB z|LMPSQMOGCA~GS88>vkjm$nl}8fQO?)n9c|sZKB6sAX~fwHdn5KKMgV{NPkZr67fY zS;drSV;1vDMWF2(FJjOa{Ac0jxGBx610Bb|SCSXgsXJ;{IaPIXGW=3GZgEW<(HAdA zAbjaw(usx!Pr`443Oa{-owruOk701B^!Dlwakkf^m(+2}M)UO)r9ftaF zm^he2ba-Q8*47$&$FvAmLEx-J8dklg?WE;aS@fdYdp4Q;$)J19AS<3(Z4WjB@?6Vi zfi>yEV9sSbjF(XLV$otTzA`V~R=~oba=r%k?n>oPtHQR4$QfwtOai}hwcSMth3ocQ z%Yn=ibi~4GrL(p7w0Ggr;v@;aaOj?r!+SD;@(6X=DdnIl#wOH_MAr!u`8UPn1gmD%TT7jAvYi=+E}uGTwr~OD%J~6-SSei`^NDR`sL;9F+cFeU zmV+tLw8u7Nf;++gChZFY1)GE>rW)x80)J9 znvf{bpN1~Y@a^RsAN_6|1vh2SQU}FVEpSe9^r$?uvFH%H486~g^~L%17HKhTRewIX zreW9Vqs3(PoAsBkikp~N9z86|N=R#MpluM0hMn2RTOM_+_hye4+67(xrlswqDZ${d zIUt4SbK=(SgoqoQCBR$rSvcMg$nR6$@667ng~%m0TpC6SZg9F{Q@gi9!H%SuN_oK@ zAxV!O8~C?7q62RE|G6{SzMQshB-4iOj4vY(qZAHeFWokH)bLzS$e*hdc19%dSdDuB zq+(MI@r$PW+*Z!kboxX&NB8*s_YKk04LQkBAx~-Ldf^$A)26oMN*7yA+S&q}oYMRt zaplUhFEyx5`9?RzH&Q_~^fT7ZzrrUUNgNzahkikxtw);FqjaThgUMPCH^%|Fvyis3 z+4Q}M9`g}tw&tZ$%ciBrtSUD-W`?;Lk<=zgG0(piQ;61gqNBl!qz*`3*}+q zT#5|rK-CY|jy~3*77=Iynafjb% z>UxD80>%aDz3aO3Wf}SPt8o2B1xaFYEA07^3m#d&c%M-5V-i&XEBNz^g*;&<;p%L; z;P2^d@+lx1G`X~^1SH*hZZGTck!?qaj<)O%ga;2)<}3>~U0A}4g>{{UEdI{ zG)19CWkyT*9XY!X7X{?QHV1e`*BaJ8V!8?+9c+6DiTY6mYjYeH;B~y&h6)-l5!fkL z&N?M*l87Nb5HU=Nbc)@-#$dtmDa`N1b)1w`{ESwI*JM`ps~>(BN1myC%%47MJuYYa zT_AZWd&-F~*L{P-!rSzH^1u|8Bn{~$&F`>l_vm&5JrPJ6KK#e9D~%qhecVMH|9thz zLF3}?Fw8(xquf-mk?(^0yz!w&#PQI^Rj#=&Kl$TcZg4AImdqhd?4o`+C0rzNDk~2Q zE_jnvjyd&)CYucI3qbpY2Uga{v#EBSzBK$=@)Dp08Ihcv2 zPA1MMGmfiHZx2mfz!!KCBFEOS#pbH}+~dOK++jG&fyP3G)XC%Zh!;7FdK&X~nP=S9 zpYnz?D9GiqRrTxq7}nLfWt|k0@(&{(^-J8UJWFj8QfrK_Yd>3^R#`Ym%o9o9OiVY| zDRsRy?T$F--b`d1K+-ZR0OVcsFW!{}wx%@UBs-p4GcFj7!Uvr3$A<#;?Yjo6o}cI| zKOQA@Bw1&r(JCq`=2qqnb}2`H30fGM2ve&$I}+B--GrD9b|HI^w7F$@d0E#(7MBV% z6cu}75rMG^zr(Eksa~O7Z0$l+wyp=m(p+HmNh2y@|C#ln-PL7Yp|d!lvw1utavBig zVo76&Aw2o8Kd(KAE%4p_#vZ7blmNwfil?rbxRn=^`IG+aDI@A!0KYsBJP(+=V=rD;%2Lr9(RS zxDG*Ho!;e@YuTv(AO0cgzvLeh;X2}*6TgP0bfE*v@2XXl`qicBY7v2g6%0&%`z~#+&W-Ocn~9o>LtfIiE-u)Xm#_M^5GdY9 z&`1x*XOXMeU7h1%_-cr`Q+JDM#Z{TL>Viz}vUCr~n5Fq#f8xk~^@~w|iT1StDq4ta zr10@;8@_mamFVbY)^~dA3kTW58|&wiTT(_e9$zh;e(GA~(XT8f-U?crUnI%5<{dp` z*}GUifWKAw@DRjqZ_A@y-0WA=uonuW1M4fvaGc(zEf1DGjnBXy&RLr@UrE8lJ&qR< zukC`9wEq%GxejbO0Lu1jf~*ZI#PT{Q-iQ+RG*e2hXlCSX&v+~^3>S%0)otY2MfFz) zUpP)fT+6CwAkk3w`S0W=fbJCNXyz4(5cp+1nnB>_G&y~vI&bjG`brfN2))8J@P{+y_Prp`Qr=0-pYW;`uS}p zXUY2jOw#=~FbPeRn}p_syQ<=vp??(J}h=K`84c(W8^4}AZJ^GWO{J$?P_O})~h>T3Rfy) zXVUK=crT|P`ND$UT3tAsBrhxl#hK~-n-LfNFBx$f=(h{|d7^-!Do{6l#4=2(rOwqQ z_R}fLUAh={1P#mX6<&Nm$!#77Jztn|;XYj#c3F>9F#fqm>fj5DopH8HT&oi}Se1jH zU5G)9?~5UZqi%O9N1NgttV;?Ut&0oR+mGnWkulmWPFQ(-71hG5zF5Mwp@qyzpd8z~ zQC92l?%6y%wv-}jffb2KmXyTw))r=XQD)-lVJ7`cS`h%O9dvLy=bS$_Mnyy$m5U+L zBhq}1rWv8B+e!53aw5Fy1wB&E_m8dJ=62^xbQ=t+8{DsSno?#wy#F=R(K^$B46xI! zcZc4r`xQE5e|FJeeQ)~whrumsZwLS`Vcr839+A6#CNDwf1OTUNCqPR<)h4Z614JK% zS6g<5K*Z5XkEAYz6Y|f_Hs&b5l$p@dcv)_ACYv8qoO)|LWF-_&emG?OYnx+G{BeL; zZ)Vlz-s%uR)7y{(Sa6;IE+oA<>NP#UF;RT3XnVm!M$B;A{Te|GFG z`JD}kz`L19cz_(YUvUBxJrdi~tD?`Q`5W=dx5Ja5J~>cK9+Lodr-??6 z)6sTrP&1aLqylBy6g2P)r&B791-!uijgETxj-v`7oehi!N+p314{#ei^?c}%jX?Y+ z$3^w8SC@J!AOg9URYPkB>n8rhd(e}Xql(&shIYkt$*_!}2r8Ko=@+L*9_Ny(XN4~1 zqmey}i+Rnj4Tr=KGtw(=Z{`6~SB)$&D?2-1x}IqmCPzi-_Tziq4Yt0^Jxxo8_>t%Q zY3fB|sF~qD(?dA)seL~k>+!G7#!JZIGmRSiFJz%*k6c6bzmwWik)U1Dn*y*bATHIsMIRWKHte{Q05js@`C zzCF?~Dln3)V_G`>MVhbsXq{l{gd#deC~X|%rT%aAEztE7=|A<8C^h640S##_9K+qJ zgrwgvyLf}EwG%xGp~9!OzndAY6~YVstp$Wb4|84~-sx$y`SoUZ+XmfyBK1)CE4c7*77<$tj8{;Q0>#cap~9H2&3L(Xi?tEspV!hB?KQ` zq}6g02SF~kwT=Dzf3uI2|0VkTo<8m#qxeg%t~RWFo8 zE%G0q&$w)k)S*_r+hkUL21OcA``?;~WO*>{)Jjufy2-En#?fmAUT&sZx4YMc+z$+f zfqo|1i$3nBku3K}QlN$GGDOEjua?$4UO!<>et~XOP7aH$hgcrx&*+}^9mHzMLfj5h zB)VdTE(K8(7KLNGGHJErdgZ1-NR)ZB^%DJjolouVd z4=qNDYiV8`37OL;O5;|V5){5a(qJ0y~~-RKs#^R#qwmSbL7$m^<_fwyB#R8d!o_6 z@9TljjA@6+#JVLl|;I*4;GN63IwR%!*eDujDR{=qzRJg^>_+0SI%S~P_<-V2F znc}qNWasG7!fBiBpHVef?frs|_lhcZyA2|d5q1*d1Y!u|M+~DcLA;Rl;7_~|B#~U3 zp7~eWXY${e_URXhAwY)(wYB1Nl-6)I6(0;^*%MqiBRQ!1 z+=drBpsLuEt=P-2QoOE)Tz?yXIC7I;D#$-FGBS$LnW*uk6K5)VW^eM>K&EP<9Fu8c zyCQ+yYvGSI%`W*rC2=7MtqYwAgF+oeA|d4@lgo@kyix$gd-N*E>`&Jnu0@D?YGMv7 zlGlB@17F%ljCy6@1rau%ljsVSSKAmqE+i~fqJsu~F!VcE{UyxDqjmM*-H>Kpae<4;i=liL;5$K*m0e6tw*hlCfB7-?x?3>B$4fmN1;{n7e% zcDBZwD6G2k@0V+bS$;4^8XG?9x=X0W#|Eh&)evc(!K|K4Y0m@&t=vlGE_(+4i|8$v zjom< zw;Y~c`BTmV;SXFY1HADU28&+;`;VFx6;cHQ@CT}PglDzvI<)IFEFkPG?uMVtKrUH1 z7F%oUxAc5*QuU#H2Hul~k6UhMQt?g3L=4RV9gQc!UAb;X%5BnZ<7*D<2f{3i?~Mki z351mBtBM4hj3;UHUyQM|>SUg|uAg5Hj%1o0Ej+&*I$0h_xj4^}4eKKA4@%R8*VUfyK(I;SBS9XEU5uOIi`4a}tXOMtevb`RAzPPgO(kQ`R#~W((0#VOJ=V3;E~U;l!umfD0((w@@)8I=ddEzI4(hzg zm{^Nc1>v(`d|}5>Xk%?fDC{eL82&R`kMp{k65p-*1&;Htm!U%M+0F$SBX)Pr`*Iv8 zmASV^pr4}?rHpi}^?Qmy!)qR*rRUs&N{c^d3-w5JzRAn$9zLBh?yNhM-B^Hl{%+U` zp~^cmG41qVM)Fm|%zVx#Tj*!9q5mBe%m1R9a^QXq}&r8qmBcaJ2kf)G?7W;GN%mAi)93hTF!@~9@iCQ)$ zi)-9m7Nacssk@&^nOC+iQ3~fqp<)R2BJuEUQ$ zRJVB}l!OGFrowe~w5v=>&leB`rU~l`U-M@V!8BYujVrFhYQJbu@=!(31|opQHBh6{ zqS#{j#t$uOJ{_UwPeRQ5_~)=$iJj&xyG9G`i;0QZbYKk^!)cZtr`)P<0;()b_5x@J zsMea=tK5|VK``X#O>}a=qkDFaavOH1-0Se6{B4McoyUpaF{dXQY*Yhc)LTW{?EA04 zoqPF|qY+0;cgT0~kO0(O5R&CP<*;=-a9MF&&9>|+BdxW7YH2E6IqW7JvLvynX)@jlqY`=wYy6Y) zt~10*-UBIb&wosL+fZV<7Z)F|)&Y)O4#??Z1iG1L+@0z+bpIgTLqV8dN)@q4OEb0o z?Nd}5H05Ix^RLU%q9-Y}oXbPsCATE=RSHriX`I8=ZoAJdX0PToxSui7x=Sr<*_Y{1 zmCY`ZuU^pNrY7{oNXErtMPopigX z<4B1FmS3#n8IR-h!rGFw7J-r-W&s?GM(GDoV%$FGrZM}HB(Hww4yp?R71mb1qyLHR zviU=99&We3Ty}ERN#Tev@dly%sC`OOiFK9S`t!Eg5fcDtY5#+eaaeP)HNFoD*(w!AvFzLwJ5RA`N>xj*p}r_EuhkY^|4618HxFO}P3 z^1Jui#k4Ftn%_X1N@~(N1um$$ZRU zl~uJi_AGyNxMcY}Xh-VJd3A5kfO#$3OpTcB8!0KD`3d8RZ2(EJ$JyCc5hw zx;Lr&p*_v~P`N5^P%G1SYyAe#irUhc-W)FvhzZ}JXBNjUS~k`J{-XK`0aY;=PZU*O zq5zfueK#&VVa?pryzY>PP86qyT&rg?Kq#YzYu9-ui;-WQSF+j}jR2#I|@GXn)}K z+uD{Z8_uZY?Me98Z5L*xi8;{bLA!&uwFd_kO$>G)vi2IzStmH2C}nOjskbJ48vnef z>>;32JQB@0;b5GO-Kc3i9q%1aYsi0^78&)fMnEo^C!vPlT9HZ>W}Sz{J8CI&q5>!H z*j0N=P9g=$rb01)fO|`89dduAP#pLF8>^<>RtecnO)S$jU#iT<^YZ7)oMR+P&(3xI zHH0HmVdYaLp!OUf`}wr-DVSTFe%oxQqz5vFnrf}eWbd+Xs`=w8fVMB z;e*!sheqME#>XDK8tYCs)+=ALjZmhG9S9cg`|N(YH9zUoXX8GlFL(pYf(ErAk_V{Q*%D?xY!*;AYrEcHppv``~za?d-G%|Q8w6E2puYaMR*p>v>I&)i_ zspUK`Gv=5+*rC7W8k%2c>qd6+uuj`)S0bvowW-H~1mT(hOXe0x`;rtg(RJzk&AZ2# zm#kusLu(=M@=bbbmrM)Z4Ap$ z@%p8dZv^MHqn{(m36t@r9y31~2XtBja-|#aR$z=t!%&MNVQBLBj>np3&Jv+qvhfwr z0ww_d1l2+i=O+L>j6;9{OCuP|#$&Ar^8+p5(thcJ!d1Uycj1C; zvg{F$o{*K}fRf{xC|)v3UDuIu4X$owXe^udh#EX;JGL(F706QX+5dEL_s)DKe_;RV zPVvrX)j0lXy--9LY#qbOCFBxN2!m0No{eyU6Kl${Q#uXM zwQ@b&RW*ez{5m?Tn6oqQ0D( z-Cw`fvV2fS={nPOt9eBVO=piGhtp2&+Rg(eP3e}20^9<<$5O@p-V8GF76I3`h;hBu zwk{H^hNE3*lXr+Ag6EP>m?^x#$cPZ*if!F^pe8#(0?ucm1GHBlOK)HL_QR918`X|i z{Tw#&H| z_3*s7qIw-$l!f>GtcbLXT;>0sbvIEo4t@9@va(OB zGZ1aRE6DMLr%Rp`JK%#h|CeAq8RNFH5lefM0)oGQ&)@z6J{!>fz!m`2QDSnd-?|*8 zmH2KC4{#X%F43j$Trcmq9_{u>;jrqxZN`o2ZB z7letM)em}NLZob7Xq~SKKmEQu=8$pA5jWM!M=f;YS6mFhp|Ss&Z7sL5_+^R<`ND;E zN+LubU7HD_BH7i!3tTWM49aHT^v!X6VeS2C1`z=3y-wte*@Dtu>3Z_chE;W7b2OK8 zae2|SxL~rxOQl#-5;(d3_QAKz_jNJn2%SlM=wf7~q*BRPHTnSv5vz~t8k#RfoDR3P zi&|zN1RJfDM^FETcBLBKg~*}X9~?z%Yo@$)T%!ShBlR4ZH$m zPKPFkNQ1L4w*y&EQrxjS1>^f2Dz1G%P9Yd>$ zaCm%mxgBl-n&{^Bf8f=OMJEHluB8@B)q#-&k#D$|xs2}g>;p_-`kni?##H(zfKaA` zl71Yt8^FLZvE=pOe$_OqcXk_qtRUns4um8y*?uJS?dBAkn81xCee*h~N=c<^14092 zShU*NCa_O;F3o*hGX6SyC239B2Mz;_r=Vslqt(FrJKC`2Cc(X2`VDrgxxn?16(GBx zg(f|)`0K7bHA`%Al?2-i_`k4CD$wD};rPq|lvo3c)tUkqa+P1O^`STmx&Cr97a=YO z=C2|`0o4=}aClDqe^YY7^)rsoe{C%a;9dXO+W+-a`=LGh+JFx5IK%?Bgj#IkX<&=g z0L_HqE$(Y*hKNrOYIM|5{9rW{AKg(FTjv?t#TuIj>#61L|DjziA(cac6zmzIT-nVK++xP78M- zKM6WQNAOj8N9-F{ymrvk4ao|6M7Xc$02jocg%W@ZLhHhpn~n}Ju*BY0XptwxB>-Fy zJIBN^o}RbCo8LaaB_K+t1-KxZ7Zud5U=f4eN$or}$M&*De_*VzUKlmn&d!Ct0Z?PD z2Y?IWuU&EjR-wO%q|@Y()Ag52F#fltLglL7Q8O|b9X;D5kW z4eA=_6-+{7G@nM8=XdZNn7V*dWv|kW{TX=QHLz=LjH~rn3ZS+5t-eI+3SEpDF~JO| zH105A{(6$k|8RB)z^Z5Qktax-;uSj_Jy$wH25>g%*nS^C5twe42vf_>1)HV?%=fg) z0kK=!^Pwrd#DSD(nmhWz8XF+cU_2^(pe;KefdT~VaM-^?b1$jE_2%j5V!@-?LI}D< z>3)FI@ZJ#YrpA6xEHd+X^{7PiR;pEv;LTDkG)2pA!k>CT_+zyRrq?tt^k}J%g3(G4 zDF$}Rl;ocXzD$(C)2t}P6@UL4W-VZe`Gum1@e%n5wDIb=Tbuyyj|YGtWoWyE65CrA zJnWB|mZ?TGP+avF#}m+=oWizlj-aB(NALe5&>ZlnT;R_@O{(Xc{e9f!!51Y9Jo>&W z$`VT_au>6NQoh=xd;_+xiN(S{u^O>Cwehhrpw`Dz7AV})bNu|@ zCsK_*k!fad7hEN17XF+v<5|tXoCw$p0T8SUF0DA4cnf{!o|yCfSO#BoQ0qhdm8RA9 zI#%l(*k=-;IoZQ@TfqU7N606K!y;46uEDfEKAvXiyyq4JKTr(}OnPj+9^pu|%g82>2JKZ^8^BK>2L{{LZ-;FA=5o5w@B%3UhRiTdI}|Fm003hj8M zQ$#p4Qi%fgMFw5X40tA;s>#FS%MF2*jdoh+9SvWf^Dp(6x*qxdWP2?kA#wiD8CXd| zLM)wMMV=|&xk%_hw;XH7+PECsx_FG`J{L$((dZFE%jO5b&tQ{$xdT>~-voj)e-Q}&i}Fo`E)=-p<`v~JY^=G5z?WKHHuH0$x6ne`i1x$ipii`B zg%Wrk3gG1dTnW(HdENsa#s6EHrUD3I)~PR$$QWdVndK14F`VQGpq9I|lS9e%Ym=s3 znRadQEV@47P)khLBU99cdoql4p;@A*`WA?HjHb)UY<2N!-%fwV(hk)pHw>w9nkG)h z_X?e8b$0*ASj&4{ca9Kn?_a@mz5>3VACt{C)`6ZQ#Q~i+Nj@@JUA&zNe(xIT94}(ub z<)*JWgIIR<83=C#okSm@2?8HYuTiC27q3vMzPnHNkr4a~NppY?P&1Kld?3KQBN>W` z4#UAIEk1p14|MJwI>CKxr+1hYUgqYl;&d#nJg+AC?oMqiM;OGQc zmnFD~?rB@#J^G4|mG2F%L)Z@>k7v}hPB;1c%FD}gb&D^el4BL9f^Q6K#S9$8o=a4l z?i}trRR*zCY0iY@>3CLJ4jIU)7$lF-FuqIu)o`7Xjt8Ozw4+q2^3H zs*RY*vJ@U(f!L@dm1^xQ^LHP@ESdzixG3woN*?;bDkH*x+1Ln@3`++;O`9QMxP!T=7fxZ5U-H!~w+<(v>T`rU_|Zk7tGXYmO__BP4&+u#_kEEau}BRdNA zvb*!}Ce|%<+eMTG9YkVa!R~pQJweo9kpS*Lz;Ljq5|z@Ard?G9*2H}gU-lYko!$UP z^F~CB1h^bs5aI4c3$TCjM4KbJ@MbP>yij^ik_#)A9q>LnK6LoMz}yY!D&FA$i}FH@ z;ySj(O8w-wd-gvr0^WATure_`lL)a^WtkKiS9ly}`{&?s^Cf=*`|kW;!H??xp= zi|0zScSY+|TjZydu`VJ}Zp8!hh9UL>7j~Vc*`_wzPQp-`|EIn83~O@h+C?D-1sezg z7MdcWbPFOiu^=iUHhNQ8f{2LJ&;yDU0UJf>q99#Bh_rwrqI5(l0tv-H=$#M}k~5yT zzVEE}z25!p{qvlC;V&+HLY_HCzsEht^!J%fVzxTn*DdT)U31^QKIVjTz``*D-$6b? z(ZJJ@UGmcUCU=)cJJV)asT1nXFcjSWGx6lbVo?`in|S;cBbkUBN?T$ed{xudj(hs-X+aaNv(V|iS%=D57n=f4qpY)d-z^-b zwU%I~>yqS+rT{N`)`{7%a5D2ika?pg*@^$kjA{KW<3-DTngRtN0#{Q z*F7Z@2rg}1Vhd{jl`i>}U(J9{2s?k6ytAiA5y=E5xxv8b6@1YssB}(&N4)&$&KD4` z4vbc$2f)Z|v@D}ij3&G$<48?*R&<~+W5#%b03<;=!x=BwATn8H9026_- zyXv6GmL!cwfZ0hoNsNkfQ>XkjNPSp{QC@iP;heHb6eaXd+MTL9`zaZQ?z$$~ zO}^86QqM6w#|ot8O^k-3c+yrZj%PA>|G&^eJwzuALx`J-vr*LF5TYIrczWWgQ>@ zZvi>b4PCtsY|-y96kW1m75h;L(HUE`<{4CC7r}co^3G`_`4Qhp-30I6GMU z8FPmO7Q?m>MaA)-`ONRv-fD+KiC2S$>E$Jle-WH=?5_0J%fwAkCgfc_MGvK-pv zeB0@6>La1XA5PZZgI5*@?&XXGbDI=WXpG$-uj^jnF}Sduoba5th~-J)!M|!q|1_EM z-C(xv_0)y;xyM5sR?Wk)M}E!IYB`FtWz16f`*+f+WUBTDd_SK)J+V{qZMR$daVxW^ z3~o0go9z8vU9FEzZ{Vs1f~Bj^J`|kKJ4lz7p;CrRGfl>uvZco~E}AU%w9p)$ZHZD7 z*(~YyHFy8{&}xOEVdtlL{$CTfoqX)~J{!$!WRp&8eR$yLDcL{A+$z?bE_u92BRxoI zWF2x#!#sFoyxE;Kp>*a{iDT}(kh0FPqK=}^v1gkJI*O$$x$y#5cj4%^cG0?z5uD_& z2|9lKgUM`pxXKZM`**?Yn$H~G%CX%rkZ`GO_!AIcAHW8}l4@c|XjO!C#hGsF4gzim z0arDw)dPL-LZBfF)+X&}#3Q7TRU8Eb47ge5GJNELhyUAk z{I{Cml1N6l2G%IvN{`K)uu^5+L>JG=50RLu79Ld{Rkb2R>F*2LP9+Ic7>sB?huCTz@Xqn2qod$hK(DKlaFooddZ-o~Ht zIm88XE2o8Cd&yUK(tX^P#@w$=WNY%L>{VsUTKmNsXPRf<`2o>W_D-8rk0fdxMTuRv zejlG4M934VY?LaRu%@BTWE~H}#iOuUy6$eC~-umwuM;!eGf{DNqKG7ehb0Xwwg74gB z#qVRk(&XcC!}MZ}i5N-Zp#=YZMvS)xBM02a-n3;5zgN8KSY>QQAr#Jh&~P%pwk69v z|LM5zsP$5$^z4$k(Wt+X(RfywRRBrSL&h|l#hBM~{uv;jK4@7$i+$Gk(`jOX=5)c- z($vkeSdf?9|FUz|5zf85V`G1^N>_HyoEFI%PmvqSHZ6;l@L~pZXQyG{5Rs`@Y}B(< zdC~Ie`PV7ybG)4wvMU{E+obQ`DM(98d;T=`6@tZuf+n~D7)Z$%a zq=L;k#u=?u2*ih+L#iCQNPUc?Bs;=p(>8;drP-<|3fzAJX0``)1enO&0`E>i9T6F1 zdl2j?7>q;ak$^a2PcUP-b_I2lAh=hgD&Xs_{0PBP0%pxmb7|aNtA*rG6Nfi(p{5ZG z`#;ZR*3Ufvjy+q&xBgArS>t1o=`Sr#ZB=u+(6}$n!&(Q)Mpsp{V;<^1EU0zObEAHd z_q^~;QBnTQ75PY2YRx)bp4l&}p3V7|^Z83>{I>QRc=Mw~&-tMVRnx2IhWi&j`(YQ> z((e`TuTz|7989T<7?p(vcGqcePCs6-hh%ppO<-}sMDl6&UF)#JR880RT^e5OBm41o z8g8Uh=0!d{!W*;OUi~Fv=qoc@w%F%*;ZFbh+2%pX?~DYZ$vxrn55F_@(@bb2A2tNw zYjR5YrZ438%oL`%Kg;g(YpleLv@IYQ_Mchbhc?-R_HHU0mW+X1U()8{PN~PdNUooB z&lWrwwn7V1kr&8_ZWzBT1G&DE;Vd+bFW!S(U%X)x4?Lj_`#hd|)DZUB6LS3yz~?_9 z;p!$`s5LN<7cq&O`yQc#nvh&SPXCesZ*V-Mrc^71?-kHb)WnS*j!IzYojiH+`(g6z zdw02R%ae|+IZ1{@rm4vwr6#k~C1!+u^LDmr?X3&h^PNq`Mn-9j9@hY!7|hC|sPi-> z?WEmO-nS)!@HV~1EOfQMp*TZxOG-MovT13cg#UQ&qLp50 z*c*RIN)#ju$4w|@BOf*2zJ0yxRXJp+Z_@<^^+ug@Hs#rPGmi1at4mnpCkpO+wsV7Y z)&<>ljZ)k9@$DtR!kwc_hcFf(A56lqb3uXZCYQxZ{TFFa;&Mp*Yt)S;VK+^X2)uLO*?V|ATY> zi_DTnN78slmh9YEhoMQJhvyghSp$9jWch~jr(h`&XWl>5Vf9Hbj>e>&CM@oekuiF3 z$$4k?$+}O!Wq&fw9>0uvrf1794hIF=jR=gti}$%1jE%c03{-jY@L>q`PFq51AMjjDU(a*_YBQT^)lT<-?JCJdgD@d!iWHNRvN zj%%!BkTtth=P=t37g&*Pay~q8jR@U0Sf4m(?QJ~KPnYzpxMcphwr}{0+@b`e66xhj z_cMlA3F*-mujKaHzpqI=DU<1R{GymX!$mI+4tR$ zhSch|ppObw@Y%d)T+-r2M?&s@(dyoV+kucvSX{P}yjBUhf43X;xW?35mkVZQCmh$b zzD*~}m?wVwM2^iAlT+HA@&a&?pN=LDHU*9?@eN|QNRise$cS^ff^{NJ3g<3&uB~aT zO^qBkh^j;y%b>UG1I;~{Te)TbD1-L0x>6cPzVyF-#-qs}}txwypbw0ii;`K`w-p5OF& zAt;@i=DB(Ei{fjp=GR`{Xo+r$$`DWR?lUW{hbr%VlGp7*U(zk_*@kN8wuJ2LFfjvP z5gi+R$GYN|D)|>Lxhlk*cX(0NfB+*3_g*2r4IU(iSW|iV{$r%M@q7JzyB0J|M!DAV zqm+;{7BY{i9NDei09f9(cT|{xhyYZIe*;v9Ady?y`^pg_mo$`fk1pz_g6LL)HEs?L z$vy&YL2^^E$lV~U7hg}Xq&4}|UIE^H&~;bw6)g z)x_nev55WC^on8wK0KT&nHvc``F62=P5AJ`T!qs5M#tpzRtbH8xtfIkqHoI9D#>!) z3W=an+0v9ny3>R4mJN%psw6dxU-IeWlOMELnhB)_=ZVbM4Zd3QltHUQ0CuJT=%OgW zy3|VZ_%#`FO6K8}qzUViiLBtV-NlL_0eJJtXD@AVR*Ezvd_mQ?Xez=6LU{Q4t^Nm* zB(C$T^YwG6-UR&80}^{yTdw21?f7EgPS*aNk^K)#SB<+#js39a%DwoaXfMI3bEBlp zCx7RflHyx?aoyJSz0Yk8Wkjm?R*Rfad|O%?J5ZNkFgD%kpwn;T^K?KidyW91LSuYV z=mLd&sTiOAyH?oKzAo;N^4s*!bA+p%{F0@bahacMu28Og9{q6nrb3MLEJcn;Ao;re zw@gDprble=|J|^f3vz%|BS=VvN8n}7N!jM*n z9C*8@W`y?vjyHA2V-7gYuLicOiVj4lAN_Xy$fXmPl7;UU`RoaKKsU|aVwst~F=`iP z()NhO{xM~(rR|D$2Tl?#o7GxF2R{Cp7~!u2 zUve3ET88go>Ww)HqvH^m3WpFmY&6UE$eX_v|4E4k^}D=qa; zZEaR*5bt7p&S69<4iIt!X@vC8^#!CUoi#z;ooBy!RqG6L3K(gAts3vmQw!fil$*b$ z#%Jml@XPD0bE>=R)DtQ{exOi&tiAQ_!1s2SQnGrar1x6Azq;G^7!=q~%_ZRR1tRO% zI41t;)O1y-$L%H1U2bF)9#(vC#^DHc#BCLipU(%AwI5dO@fTQ-gmAH8HKN}=&Hz$R z6YwgU5O9t|n0RvP9gN_tV8b3TIpGPLs07+ttTl9?fc4#X(qb=(d0@^ow z9`dmzYtxc#*Q-aiX4~k~FabH)#$vDOw%M?%dhpW(?_*atXu>*e;bNLsvcT-1nkC8b zF1!F;3YcF%0C_AI>gb1o!yBP;K!-w3+N5=t4jKcOpDBHYi&wBsVD3Eiw$X4&57)HF zWiIq(1Q{Q#yNLz`ss@CO*3P|20s^uUtl8yKc1@0AH>i5e!!tze z*L(lu%Y~s2)kM^ODIIsdsY?c5aPF+jwsiW+Z!XdDyr1E!9rC7&Ne(|i4xxYqemTTN zd+36#+$Pm?1Tu`prrptk2t9OhnPT8@z(of!GpZqqW=7-Zv+7;6(w$ErVFKaKblu8 zHgH_ype$K8u)+&H+v=5f^Jy>I=L5CGyJaAc(ymB}&7U}bT8s60FMFx5voInfr=-}@ zqdtQ2CFX{L8p+SLXr@8)-t8A(pQU)2-RK^TqB1OB;+1|qN8VxiFF32F0Muu zJFedSG6<{0T^1ql+#(o#E0FZ^@I$+|iaoEKom?0jqy$7JKk*CsaB-v7q1{bZxLXKH z2?#b>Y|{zRlWVHfCCe1kdB~M^qFQY^HYT1EBrDGboj0C?4RJ~fO5BPw&QX-eLsK#+ z^fBecl4Fu*qgJ^$6KdOw)!!!wmprIB#5ZbEp-WzxUO0k6^X?a&WV@+`S8($Rn7#=l znae~w%VIBXgqPg7=OCIJl_Ah%kx?<7h30*980#FmedRssDfn~)%FPmmJ~{P?MhN5z zlHhSS2~Shs7u1FFk;k#l#}QLQi0A8j&NNW+x{`j(T}PqUM}R;sar7d^3f=(c>uyb=p67MN< zth0fJb>)WW%)s%N*Wa&{Rx#r)Q|-9A?%2gVl#gRx^OjM8)gP$7ye`@=xq^;J_uRo> zwbYJ)ALSEE)@S9U3c)o&fut>@A3~j=5uG}>k9eb^DC^&3+Y3qEe2aE(OL|(rWGU+J zHi&LIntS3{^8p2H$aVrPgkqye?Y*00- zIHP}3EDn1PlyJY7tq|1`i}R33>=!z|E0G|0_r_nSIC!>Ug}Wc>1PJ$gPpCnnEqtSk zSJ-K(v_R6yEv30f;2ZBQDD1>0-U$>9SmWUf(h_d6I(uk@PvEU$BH>y{XE-3!Ag$Hr zhR6$d|MP{-QD}2MYPRw8ybShY7HnviYwHH7rjuCbf~N#!Zqx?uZsF$qNUk6e5ayj? zZ9akfywt4w@dai)mb3LQON;p26?*QFIWLq8VihAc^SR3<5;WE|9#o z?7d}*UjO*7SpWh-xx5QzE(#|Ot^#uab`f$6gA1Q5D`^)tm^@$>pVH z=kDkg0|%5DNYXIfP1XynFa-x>`t)X4JcxE|x1v3)cU#Y$31t*FN_#amTjIQ%9jb_D z)_9z0fDs30mip|?30{Hdcrc8XTDVHSi0X^mP@ZrrFhMx=@vF_ezlEBt=zG=n1d-rF z6MPz&AU`lWn}MA>Rd~3&Rqt%9$4W&7l4>Fd4t%HJEe*Em?)xEfvzbu%VTo6Oca2x? zJw9zPx0Rp-+{KD~aK9EcOCX^NeN4LN)x_0THw8fXMvB&SHUMd;+G=0>{mTVsznm8=EZFO*t7{#KOu~eV*L~-@R4=zPln(+AlVQ zD807o$tP5R@!J3B55X-+B@(tIc0T0_dKWmpn)6TuY|E9KoroW8vbc1Z|M)I&te=`f zPbK0YK5)FTI_Pvmmw`c|J-n-%Ps8b+3{d8PpPzDgA&eY=SE-KtIyvaljPp8gcwM>2 zzG{B+L2eMgD2QJz{}xwJA!ub*c{ePj47?_1dGFyr$M)Ssq0q*%i3H<8N(sE4aw6fE zeGhtx4?KX4-RV`j0URL1M2HqVTo2>YbsO=4EfJF{+Yv=?8gX}L+kL^_92H6J*gkq&$ zfO(31aN#?J1=&j-mA)VfvezhD2Nty2t5=UFOBS)%@oPyp3{mLq@bOpnia}Gn5VYbI zVuK;-;NSza@$D^)GDHJ^9DTE3)=Y+F`M{TJrwr7gUBNeW(PQF48GJa(z{TB7nSf80 zfhLgz-K{K9j1qhF;I}*(s z4C3TG^j0B}Aa}bn4ABM>Xan{z{5c`Sot1&w)8Z3ngucSIyAIsAhSG*b#J9M4qHKg= zxfvn>x?mHzAScbjqjxdkA;iiUpUb|W_JxfVd=MSWKxO~P8QiJU(%5N4wP#!ZED!$k zT0)SJfYdt;?y>?6WwJpa))!t+XN4@)5CRozuiItR5Krl33tN~Ao2U=I@XGb|22uqe zEAjGP*&VP^!Jqc8=0-gef?(wOWPIP_5F+F8!Y4GEnR$>S*CgYo@By6Y3(|zuY@Bt@!>tH_B$V2 zV4W!qv5E9VmTeumt#=6|h}@ z1P$xE-*_2?-U6{;CADw|R!SkUQHh+jX6|X8zxRL+4@nJ zeC@}9^j)xqhwX*iu@_aqL2Y`oYJ=376If@XTZQYmQMX~oKJL=K&lU6#!pio-EC|a- zvCiv0QMaLaFMx&#=Q^owMm9+3%^}|38zw(BJ}WrEcj^QzdbdHoz*k5LU4LJ%8A4R& z)EC)=09WrXc=}LYK;>eYgx5z&wi&&mgk0X%#=Hv}wn|>5FoamX;MSFH5D_r*w?t-U}av=XRhK#ZGuh$KH39EoVGB3P7D9mS>FSOza^MF{pSdW3f^D4qWb@>UB*df3 zuyK7i#X`A)u7JbIQy?Bs)Py~`_uzVeN+>0h5nC{L2{XOVP%(y|Y)P7~QH!FdE-v1K z1v_ro)TvAMC65GLSt`z9u39ItWEHM#AMcVgJ!RXuzUk(~FKlQar_S8(!^=(N`niF^2Ts1OBnc!?Qza13iM#+OdNBrd%#}%{( zYLPvu{nydF3Xlv$o}W@hp}#l8HwTEam}}U!9X_90 zZx>Tb9A;8n2h{B&&NpfljDGOfD`JL3Md&LRyH^cLj?xm`*o^0NN`GSjb6<#kLO|ob znkRaaW=iD?)2`nJm8?e#`gXFwjh*E^stDce&OYP`*ls$=N4(By`Aj#anyUDSfbpzgQck)Jd3$XC5(B zCQr`sW8Q`kXY})FS_~qk-L_Cc=|=!Py|(sE{>=4~u~z?v{9nqMb4-1P7Ax0KksnW$ z77_Jkx7Avo883CC^@XpGDOyS}^f}g1O!?sYvD4ZnyPWm1s!h#V=d$OX1^e2wdgA9v zw2!ps81ETLMeS<_Y)}b<>ms3+6^#$2`M*v<2sg&!6l?oJz}Lg=+}^uSo5s4Gwi zQ5KX^aRoD*4itl$#UE5oX9p2H|5q@$uPaVyr5d3XPt zc_%6(VXwh4AZh*qX706X&5P8?$^#O{tvNO}o)f=R8Glu4)%jLw4EgtFvPb+{5E z@OBy1msBZXDlz3BzU!W(i`R<@0;55*Hf&6@rohDdJZT|M?YYH`;ww3WIRlIYIyv6x zfH*PDa=Ig=c8I2td_8{2XX);`kU_G~j3Q;7vj35y#fKU5K1A@{DoQ~Gmb2P(&-c~Z zmeamb+B+z}>SNTRJo4Ecxij;wBZU^VJkI`3^uDTuPRFm+OCyrIZQRDaDdKqgK%U3Q z6s2^P)@w1Bg(p{9z8K`|=iro-Du}}c<>hTJ>CWp1UQJ9@$HALtg)7mtMu;KyH=;e7 zMh{*Y3=^jeSUAf$Gg~lnHbhqn``t&K7^Rsz1@U;+TZ5PbQG+^+w%R>$#N&k^Kpg~f$xZyS`x_Kc%^M#i@XKqA@>vV{*5_aL(U zav}Bs$QBahiKuEx(1vp$+wT{}zyS%va}W2k0uIGypcluo;-n{8PZCQ8gE z*%|}2F3YeES9kS6NAMqYI-6-LCIZ-fLcUSHpFUd7`Ztv?-kr8GTo})rTgaj;I%X@| zEHY(6^gU?!$u>nLVLa1n_d99qy_HM>2#)P+Pm7F2ucezxH=MUl?Jd?cK z)cZ;HN_H>ta#Qm75n-JUT=+X1flUb&*OV9I1BT5g_W~vz*wYTBgecgcA=i;C!gEG9 zjhhnPkve4UF(*t3w_ft4ymZXz+BX`mL*Gm(&G9{6)ca+m=1T{KlpLm2+~w6WN2!rl zjw`N)|KfQ3;KRlqX@9-EU-*7F_b^Zi##_*H&Aj#(Z zflTrlrLxfw3Xa{Dqp0m_zy|1UFh-ji#Ghbg%KDb)&v2sPnFGqtI}4pP-|o{IVY4WE z%g-zsf*nl%QY)QJY|WfCfIZ&PKcc+`vM-oX9*4Dtq)pBDnGn?olD3zzZbDz>6A9CH z&aOxXpUuhO)ghJoXE{eY2@duTvQQ7Xk7G*yV>ny~8OAnZ7N__maf;7lvJh=+o@gzJ z;?uikx_l!?vWeNHv);-|CFAWz2gC}K!~%YApmfbNC`#hl)glR_C0#;@%HtOb?4x{R zJ;PloZ%iWFSgefNu})`YO2fH^_%U_%j$W&LL;irNoTWoWqajg>g{EUPWhrQOg~rB1cIGy#zMyyVkoNS-)oUs({eU83xM^ed#HT@>Uz5(--Hqz*O z)w>R-eQADoQAzEZfRU&grjT3PVxxH@hCD%LwdadOC1R#?X%TI)?5DRiK4~`+4ZRn` z*_P^~?FzL9r0pM2_lzS|9=4ME8AfK~6RrP{bpgkdw^eZ6@ z1MW8yqTav9&RmFpNO**SuFbf*p@UnIkm%K`{Ks_zQWdHtRSln&1lP^~Kv{r#4#XW{ z>pRV!d+v%j((!h0J^JTSA%Eq?AEGsHME%n&UA_rZ?sr@vcNTUR_Vul>n$;fq;rra4 zMPtAF;Kd*~`y0|{^)Lm4W^GEW39PFfxwi9?5(g_VU~xFseI!@8 z(e0_PKJ@u~%{SV?tk|2(6ZwW>I>}m--27&vlj&`&0Q!Qgj(cFYF{>Z^>qx6mO{30C zd4xf};hunRlB{F&E>WGL>7DFp*=kALW$)qh0mZQg7{7nhQ-#W@RprDFy#5t;81$8* z{_LAAVZCqO z8jB?W7SKr*D@}ql4MB7QD}JF%C))<>gt<{pP`_B#qh>@K6ESk#E>(CK^26HEH%#cOqXu%iBwi}&wERxd5|i#j?N z_oh_MOe`k$8)^hhb>h9+d~12|OnXFBHl^;(G*x9E5Bea#A2T~#xiNs$7%60^lk3;h zJ)w+We5f`yXCH6`=?oad4hQhf6*aQv@@{k&yE4W)YqJQX(l)6X!?y1C4sJ!vhx#?n z2r(&JxrU>G6+vK9LBULQFe#`YZ1q2T^9mdU zR7B(PV+^FDkHDlJr5zOFMls;O87))x5C8$B3*94ZzXhr9Z4JND!3gy#*5CnL0&oVY zuMZs$J04hJ4XLd1`tof}S@4{^9^uAPvFoQ3vd>$_y?xYaK{WZ)I1h96q4&>fiaXeK zn0i`Xh?1f*yB=dX&%LUc%6RPTdz}6;Rr^w|XMls(U|3wO&fq70{42jnZApcUB^JGM z9@k70MO@1si!d`wD(&;-@H0U&--HndhgXxqHNA( z7mcS++vXd7(VqUAT)Wjy)E}b)iuW^~t&W+=2r=Bm$|$GLOwZ%S*ch*ypGTig@Uwfu zqjJ<~rLKc049dp(9sc8`=IFBX0mIS>U4y;@HthkO^RAs)govL3Y`?LSqW*>&bFZiC z*sRRFfThLhVvLs)Y_pCA+Y^aMhY`_Ol?NQ5-x<`(x!&Pwi8!1KEauptvSxF(=J+nP zZokTSj0LCwOu4o39@`40MU$;%w!k$Ubi>G~Ql{Fx)&mHyYg}CLgbQZg9_8Ns?8CyZ&Si;Xc0=WY}VOuCJV`YW=1gw-(70dCF7T(Wg~JJzg9(P+KE1~ zp?!&pxTmvVQRCl0|J9K?0I@S*DWtaCXFWMRy@xh=V!AS=N z!TSPE%%Du&1MhnVysyeey)6iY@pu2aBv6#qt;>vu?kNDW{JYhqL8agmxSx}bs3j;# zO8h)&+7n&V$(Whse8aJB?Gj#J+oGxBn3{Sul4Y3I;#b%3mhs$$_arC6b$MmK@tN8& z!WbhhEZl#X|M@+9wVIMn!SutBoM?Z!iNZXOdA{-qzkbacS@zJUs+5`9(s18j{k6S% zV%FK^1B+E}`Yh~x#quf9u3aapqdZ1hBSe>KCL)ot(|}y=-i)gr(-evF!;P0Fi+Q$R zqP#wDsIb+J&5Bt=vLew_aCJ&dQjylJ96_lj%Zmk~{)3|D@xM?~#jVRlAI~2_qXVgMW zRY+K@tFGxJ&!y0#YFpwL>pD^=b#6|jm5SnrX0G&qsi(Y^4wtJI$6f9y9h$77Dfqvd za7$Pkp#=9En){92B9sxe`Yed4Vs5jMRTE^tj~TVeFyK1BKu%e$3lO)1Ps39+JY*y_W-4qIOlo{g&v2_5tDkRIEeS43>hhV%q9n5yo=k_$kckT$ztL|&_<+nfL#aGH_GY+{NMjskFmkK& zZb@IVUTul;!bp|QOb#7h-DPzqWJ|>!Bef@F<1L93&4vp0^*VPvS$rk9Ivs8t>s!%` z;jg9NsTgCVs!>*AD;Y2sI#N4z{N{%WGnS5$E7c^=O3>OR75OzLE`_jFJ~E$qsnzLZ z`LP}la@5%}A=L@4(?gU8;cNnBwxz*(6oVl%g#s~CY2nX>3=bA{jf^rPO_);?k!$SB z@kk1+)x+Yq_PaHqZ!;A`=`5eyL+N7pb-!9+Gd)c<^;`Io##C>DzqzHJN>$bpPLz`WWl5RXIrFrCr&{p2p5p+RD^HMvN1V*a46W7 z`AzmYorgr19kQOGWYik8k>3vr`3#N>;0pn_xY9oWYB3ducj~O~ta0kRAx~(~`Gy;6 z@f);BVCTP_uz}!jpW~xn^xI7|>U@J%!S!+{-=gsvN72SIMt*tTZMHt3-)LjsX57%| zjA5O&-%Rh7+T7~;Mf?7~ya@v}@|i-Xk^+Dr*5$LgK90X}R)CdhOF1ZxohTfKAr1 zn6p(p-EmrrTCZ{Ygo{ zOitwidR*l;(O$%){9Z9O*a##J<#(+iM19%rcfId)A=@_tQzPE%OQ2sNt=mfc?lHq& z_-LiPL3xZYt9@nu+kpJiD!~D=CT61Vd=qZMT%n9HG01KXwgOWquhpfz(Oy45p3(o- zNPl5tWqFcyJ4(CUa<-~btS;}!qBbDi=dAgicR;)7*kqv$RtkEBhy6r^q?#b3R>cxz zxKZ1<%Y^&#qq%~P0hF_$)CpWDq-f^*;_d*C0Gw%Pk5gv>L|Cip!AZ0$AO?eV-9OMi z(3UTgr83de0M7fpi1B&}`+9m`8~p+fIt1M(431qaMm0mHJHo8M<~Wp@`dpQj-uAYHfH>E$|p_i@K6p6$r~@sX@M07US3JO6I_tmXL5>TA?u&?E4&A zRAkrgxwhvB^mj>;_$9T7T5{}Q?J-5z(X-Y4KtIfc7P@iZ)D>Wc0fcsJYv^rAcY%Fo z&iy5dPhe94ysU@TdSI;Vg>-lGMjKGPAMn$Use@g|p_wS|pb`dvP7M?-@|_DpkPZUT z=}S!LGx#e&bTTJZneff-1g?ei197G|t=zp4f?pO6V#oY{CNTUr{E34v_r29%CiD7LYG=a@bz_bx!4IE zvI1a#6%?8224Xd#sog+oK;PX6t3PhgxE&fV@4!gckEkPBj9~M-(vLuPpea21E+PykW&50w&}F0{V)AzXkx9{Ygj23&v{F7X6l zzIFd$zCkOvbN^(%aTag!XYr6E=yI4m zg@L4W#Tyu;%kKk=_df<(y)Wg49W2AE7ssLW$w9J#EKD3&31O9P?m+3AHRe$I+(2Re z59tKR!m?puFR+7TFzf~pEbf6`uj4SU3P8Ay9M<19uwA5S8;CUkTE_WKYtHHk zf3F@k_*dV&G*$|>{MD^i@bQPSxPmhT@DI>7{&(6Rd@o1xKR{0ls>Xm;djTZRlhugu zy1x9VKb7bP;qiawDk7wrlK(@}Oc1b!x{-w;tVrj7vLb)K1Fry#GJwnqfzQOzFqL3% zn}?hCaaz`de{ER@Vx73VjX1JD3R`}m$p=V}fb@tA&+ouWK^uI#Pb<_*K&<_<__M%y zL+fi9RO;W zyUG6{HB12Lo+bZE_xxMw#Q>#JJ4NM2^Zo$VUJggY?tI@XF>vyQuony% z$)D9LLc{)l%lL$X=+ELoCpnS_wRirf+Uq347H>YUwhxw3I;qK_IRVtnSy&i+Yjv-> zM$l_eb%)6>JDZBY8EIjwWx))cIcB&MLflRcQ4WIeH@xkf+ zg3!j%unGaNcwXH`5OfOISDKSDa1$W({8@bgZ16w53x}Niuf&>v@^j&7j^wX_`TbMv z?ZMrt*$&$w%McCVNbEH@FwVkq0Zrs^ctO==){)u~|6=t||MH>&4JL*@ydM?x7wm>v z&leeB0G6r#k4!OFQ^*%ol2#$!2K*pL!#08GtxKYA20?=(S1lNW5HA2x|Ig|R1IPdA zUH%h&>>teRw?IzjaDAP?=>MVii;rN7?M`*q!7_lH%#j#4us;g}tLCt`pMt8t{{}KL` z*WuszU$X#~q2T{+G%Z8H%TO?kSXl0_B8}ta{_1jn^*@c?SnjXFz>4Mm>T-V-p%Egk%cP}c($X?%X_>UNOj`O+L)@18tIPe><^Jk&e|5RP3N+|tCh#&7c$o>j%miLO zUbB3>X8Cvx3{+n}Uh`kG|NmwD{qpge<>NKW$7`0)tp0})*vn^D@7`EGv$}j{6;AB_ zUvOr1l1*@4rN&h$wU&!UEDz;Z3+|dMQbt6_fJE88uZuCxS#bIF!<<^mY zp~E2TaCQL*&_#*mhxdW}EY4<^!YNqgjopU=D_+B#g&zlC2(u`RZNBqL6D~=B zJ~C$`Kt@>{86ALw(3}V9Wjp7N$XP9;t?nn&Q$)9++d{lSEyQXq|i8KeP^Hv<_yDa8~pHly^23t`Itcb^fEgz&_%L z8aBq_9~(nI#*JA#kaEKwRn6l!8ICC8NzgKf$=01WVdRDQ`o95Q15;2|A0fCZ^yrbJ zn|Q$Xmd$b59KpdXo8z)ME=NZsM_7)IoU~^-Ixa`Y2i-uT9q}11dR$zqT2CB3bYU6RS#EkP sH$9enkjp*D<-~QF67YWkB|zuU{jWQ*>(7(8;NJ<|(?_!pU%d7I0Apl^egFUf literal 0 HcmV?d00001 diff --git a/mix.exs b/mix.exs index 111b3ab1..77c2719d 100644 --- a/mix.exs +++ b/mix.exs @@ -53,7 +53,7 @@ defmodule Supavisor.MixProject do {:plug_cowboy, "~> 2.5"}, {:joken, "~> 2.5.0"}, {:cloak_ecto, "~> 1.2.0"}, - {:meck, "~> 0.9.2", only: :test}, + {:meck, "~> 0.9.2", only: [:dev, :test]}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, {:benchee, "~> 1.1.0", only: :dev}, @@ -66,6 +66,7 @@ defmodule Supavisor.MixProject do {:cachex, "~> 3.6"}, {:inet_cidr, "~> 1.0.0"}, {:observer_cli, "~> 1.7"}, + {:eflambe, "~> 0.3.1", only: [:dev]}, # pooller # {:poolboy, "~> 1.5.2"}, diff --git a/mix.lock b/mix.lock index e00e6a6c..a545bc4e 100644 --- a/mix.lock +++ b/mix.lock @@ -21,6 +21,7 @@ "distillery": {:hex, :distillery, "2.1.1", "f9332afc2eec8a1a2b86f22429e068ef35f84a93ea1718265e740d90dd367814", [:mix], [{:artificery, "~> 0.2", [hex: :artificery, repo: "hexpm", optional: false]}], "hexpm", "bbc7008b0161a6f130d8d903b5b3232351fccc9c31a991f8fcbf2a12ace22995"}, "ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"}, "ecto_sql": {:hex, :ecto_sql, "3.10.2", "6b98b46534b5c2f8b8b5f03f126e75e2a73c64f3c071149d32987a5378b0fdbd", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "68c018debca57cb9235e3889affdaec7a10616a4e3a80c99fa1d01fdafaa9007"}, + "eflambe": {:hex, :eflambe, "0.3.1", "ef0a35084fad1f50744496730a9662782c0a9ebf449d3e03143e23295c5926ea", [:rebar3], [{:meck, "0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "58d5997be606d4e269e9e9705338e055281fdf3e4935cc902c8908e9e4516c5f"}, "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, @@ -84,4 +85,3 @@ "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.5", "9dfeee8269b27e958a65b3e235b7e447769f66b5b5925385f5a569269164a210", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b977ba4a01918acbf77045ff88de7f6972c2a009213c515a445c48f224ffce9"}, } - From 2d5744edea2050cd0f720bf7b937d8f1de439f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 2 Jul 2024 12:29:30 +0200 Subject: [PATCH 03/97] ft: improve some decoding paths (#372) --- lib/supavisor/protocol/client.ex | 16 ++++++---------- lib/supavisor/protocol/server.ex | 4 ++-- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/supavisor/protocol/client.ex b/lib/supavisor/protocol/client.ex index b9a3c966..dc78e08f 100644 --- a/lib/supavisor/protocol/client.ex +++ b/lib/supavisor/protocol/client.ex @@ -103,22 +103,18 @@ defmodule Supavisor.Protocol.Client do end def decode_payload(:simple_query, payload) do - case String.split(payload, <<0>>) do + case :binary.split(payload, <<0>>) do [query, ""] -> query _ -> :undefined end end + def decode_payload(:parse_message, <<0>>), do: :undefined + def decode_payload(:parse_message, payload) do - case String.split(payload, <<0>>) do - [""] -> - :undefined - - other -> - case Enum.filter(other, &(&1 != "")) do - [sql] -> sql - message -> message - end + case :binary.split(payload, <<0>>, [:global, :trim_all]) do + [sql] -> sql + message -> message end end diff --git a/lib/supavisor/protocol/server.ex b/lib/supavisor/protocol/server.ex index ac96d571..c2986e24 100644 --- a/lib/supavisor/protocol/server.ex +++ b/lib/supavisor/protocol/server.ex @@ -172,7 +172,7 @@ defmodule Supavisor.Protocol.Server do # https://www.postgresql.org/docs/current/protocol-error-fields.html def decode_payload(:error_response, payload) do - String.split(payload, <<0>>, trim: true) + :binary.split(payload, <<0>>, [:global, :trim_all]) end def decode_payload( @@ -195,7 +195,7 @@ defmodule Supavisor.Protocol.Server do end def decode_payload(:password_message, "md5" <> _ = bin) do - case String.split(bin, <<0>>) do + case :binary.split(bin, <<0>>) do [digest, ""] -> {:md5, digest} _ -> :undefined end From c1d9f9ad3f1569913f00a0c5e43831ab158bb9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 2 Jul 2024 12:42:49 +0200 Subject: [PATCH 04/97] chore: use `or/2` over `||/2` (#371) Reason for that is that `a || b` will expand to: ``` case a do val when val in [false, nil] -> b val -> val end ``` Which is harder to optimise by the compiler. `or/2` works only with booleans, so in places where we do not care about returned value or `nil` cases, it is better to use `or/2`. --- lib/supavisor/client_handler.ex | 4 ++-- lib/supavisor/db_handler.ex | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 4c46f5fe..146ca166 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -297,7 +297,7 @@ defmodule Supavisor.ClientHandler do Cachex.get(Supavisor.Cache, key) == {:ok, nil} do case auth_secrets(info, data.user, key, 15_000) do {:ok, {method2, secrets2}} = value -> - if method != method2 || Map.delete(secrets.(), :client_key) != secrets2.() do + if method != method2 or Map.delete(secrets.(), :client_key) != secrets2.() do Logger.warning("ClientHandler: Update secrets and terminate pool") Cachex.update( @@ -632,7 +632,7 @@ defmodule Supavisor.ClientHandler do db_info = case Db.get_state_and_mode(pid) do {:ok, {state, mode} = resp} -> - if state == :busy || mode == :session, do: Db.stop(pid) + if state == :busy or mode == :session, do: Db.stop(pid) resp error -> diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index e14e59d6..acdb1141 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -451,7 +451,7 @@ defmodule Supavisor.DbHandler do ) end - if state == :busy || data.mode == :session do + if state == :busy or data.mode == :session do :ok = sock_send(data.sock, <>) :gen_tcp.close(elem(data.sock, 1)) {:stop, {:client_handler_down, data.mode}} From a6b4a0f3cecb9219abdf2d3512ba6b6373e4125c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 2 Jul 2024 13:18:28 +0200 Subject: [PATCH 05/97] ft: do not traverse whole state for actions handling (#377) --- lib/supavisor/client_handler.ex | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 146ca166..cb1b5a49 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -931,11 +931,6 @@ defmodule Supavisor.ClientHandler do def try_get_sni(_), do: nil - @spec timeout_check(atom, non_neg_integer) :: {:timeout, non_neg_integer, atom} - defp timeout_check(key, timeout) do - {:timeout, timeout, key} - end - defp db_pid_meta({_, {_, pid}} = _key) do rkey = Supavisor.Registry.PoolPids fnode = node(pid) @@ -976,18 +971,15 @@ defmodule Supavisor.ClientHandler do defp handle_prepared_statements(_, _, _), do: nil @spec handle_actions(map) :: [{:timeout, non_neg_integer, atom}] - defp handle_actions(data) do - Enum.flat_map(data, fn - {:heartbeat_interval, v} = t when v > 0 -> - Logger.debug("ClientHandler: Call timeout #{inspect(t)}") - [timeout_check(:heartbeat_check, v)] - - {:idle_timeout, v} = t when v > 0 -> - Logger.debug("ClientHandler: Call timeout #{inspect(t)}") - [timeout_check(:idle_terminate, v)] - - _ -> - [] - end) + defp handle_actions(%{} = data) do + heartbeat = + if data.heartbeat_interval > 0, + do: [{:timeout, data.heartbeat_interval, :heartbeat_check}], + else: [] + + idle = + if data.idle_timeout > 0, do: [{:timeout, data.idle_timeout, :idle_timeout}], else: [] + + idle ++ heartbeat end end From 61b48dd66fc19aa3547aaf3479747f2a7e33cde7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 2 Jul 2024 13:59:37 +0200 Subject: [PATCH 06/97] ft: gather metrics with low priority (#379) * ft: improve socket metrics gathering As we are only interested in just 2 of the values from the socket stats, fetch only these. This should reduce time required for processing data sent between port and application. It also simplifies pattern matching and data extraction. * ft: gather metrics from other nodes with low priority --- lib/supavisor/monitoring/telem.ex | 12 +++++++----- lib/supavisor/tenants_metrics.ex | 7 ++++++- test/supavisor/db_handler_test.exs | 4 ++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/supavisor/monitoring/telem.ex b/lib/supavisor/monitoring/telem.ex index c983925a..460b7524 100644 --- a/lib/supavisor/monitoring/telem.ex +++ b/lib/supavisor/monitoring/telem.ex @@ -9,10 +9,12 @@ defmodule Supavisor.Monitoring.Telem do def network_usage(type, {mod, socket}, id, stats) do mod = if mod == :ssl, do: :ssl, else: :inet - case mod.getstat(socket) do - {:ok, values} -> - values = Map.new(values) - diff = Map.merge(values, stats, fn _, v1, v2 -> v1 - v2 end) + case mod.getstat(socket, [:recv_oct, :send_oct]) do + {:ok, [{:recv_oct, recv_oct}, {:send_oct, send_oct}]} -> + diff = %{ + send_oct: send_oct - Map.get(stats, :send_oct, 0), + recv_oct: recv_oct - Map.get(stats, :recv_oct, 0) + } {{ptype, tenant}, user, mode, db_name} = id @@ -22,7 +24,7 @@ defmodule Supavisor.Monitoring.Telem do %{tenant: tenant, user: user, mode: mode, type: ptype, db_name: db_name} ) - {:ok, values} + {:ok, %{recv_oct: recv_oct, send_oct: send_oct}} {:error, reason} -> Logger.error("Failed to get socket stats: #{inspect(reason)}") diff --git a/lib/supavisor/tenants_metrics.ex b/lib/supavisor/tenants_metrics.ex index 6c2605e6..25f71083 100644 --- a/lib/supavisor/tenants_metrics.ex +++ b/lib/supavisor/tenants_metrics.ex @@ -8,7 +8,12 @@ defmodule Supavisor.TenantsMetrics do @check_timeout 10_000 def start_link(args) do - GenServer.start_link(__MODULE__, args, name: __MODULE__) + GenServer.start_link(__MODULE__, args, + name: __MODULE__, + spawn_opt: [ + priority: :low + ] + ) end ## Callbacks diff --git a/test/supavisor/db_handler_test.exs b/test/supavisor/db_handler_test.exs index e2610de6..417b72f2 100644 --- a/test/supavisor/db_handler_test.exs +++ b/test/supavisor/db_handler_test.exs @@ -192,7 +192,7 @@ defmodule Supavisor.DbHandlerTest do :meck.new(:inet, [:unstick, :passthrough]) :meck.expect(:prim_inet, :getstat, fn _, _ -> - {:ok, %{}} + {:ok, [{:recv_oct, 21}, {:send_oct, 37}]} end) :meck.expect(:inet, :setopts, fn _, _ -> :ok end) @@ -224,7 +224,7 @@ defmodule Supavisor.DbHandlerTest do :meck.new(:inet, [:unstick, :passthrough]) :meck.expect(:prim_inet, :getstat, fn _, _ -> - {:ok, %{}} + {:ok, [{:recv_oct, 21}, {:send_oct, 37}]} end) :meck.expect(:inet, :setopts, fn _, _ -> :ok end) From b436f9e9c81b5669f9efc195228be72b80ad7161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 2 Jul 2024 14:19:51 +0200 Subject: [PATCH 07/97] chore: fetch PLT cache with relaxed key as well (#380) --- .github/workflows/elixir.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index cce7e1ff..bede91b5 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -171,6 +171,9 @@ jobs: with: path: _build/${{ env.MIX_ENV }}/*.plt key: ${{ runner.os }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-plts-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + restore-keys: | + ${{ runner.os }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-plts- + - name: Create PLTs if: steps.plt-cache.outputs.cache-hit != 'true' run: | From d7ba466c6411418da54298dc18f2a7900d09d406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 2 Jul 2024 15:56:26 +0200 Subject: [PATCH 08/97] ft: update Ranch to 2.0 (#382) --- lib/supavisor/client_handler.ex | 2 +- lib/supavisor/native_handler.ex | 2 +- mix.exs | 4 ++-- mix.lock | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index cb1b5a49..96ac9420 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -18,7 +18,7 @@ defmodule Supavisor.ClientHandler do alias Supavisor.{Tenants, Monitoring.Telem, Protocol.Client, Protocol.Server} @impl true - def start_link(ref, _sock, transport, opts) do + def start_link(ref, transport, opts) do pid = :proc_lib.spawn_link(__MODULE__, :init, [ref, transport, opts]) {:ok, pid} end diff --git a/lib/supavisor/native_handler.ex b/lib/supavisor/native_handler.ex index a02255ef..a3de4d0a 100644 --- a/lib/supavisor/native_handler.ex +++ b/lib/supavisor/native_handler.ex @@ -10,7 +10,7 @@ defmodule Supavisor.NativeHandler do alias Supavisor.{Protocol.Server, Tenants} @impl true - def start_link(ref, _sock, transport, opts) do + def start_link(ref, transport, opts) do pid = :proc_lib.spawn_link(__MODULE__, :init, [ref, transport, opts]) {:ok, pid} end diff --git a/mix.exs b/mix.exs index 77c2719d..6e1dca6b 100644 --- a/mix.exs +++ b/mix.exs @@ -74,8 +74,8 @@ defmodule Supavisor.MixProject do {:partisan, git: "https://github.com/lasp-lang/partisan.git", tag: "v5.0.0-rc.12"}, {:syn, "~> 3.3"}, {:pgo, "~> 0.13"}, - {:rustler, "~> 0.29.1"} - # TODO: add ranch deps + {:rustler, "~> 0.29.1"}, + {:ranch, "~> 2.0", override: true} ] end diff --git a/mix.lock b/mix.lock index a545bc4e..e26edd8e 100644 --- a/mix.lock +++ b/mix.lock @@ -65,7 +65,7 @@ "postgrex": {:hex, :postgrex, "0.17.3", "c92cda8de2033a7585dae8c61b1d420a1a1322421df84da9a82a6764580c503d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "946cf46935a4fdca7a81448be76ba3503cff082df42c6ec1ff16a4bdfbfb098d"}, "prom_ex": {:hex, :prom_ex, "1.8.0", "662615e1d2f2ab3e0dc13a51c92ad0ccfcab24336a90cb9b114ee1bce9ef88aa", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.15", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.3", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "3eea763dfa941e25de50decbf17a6a94dbd2270e7b32f88279aa6e9bbb8e23e7"}, "quickrand": {:hex, :quickrand, "2.0.7", "d2bd76676a446e6a058d678444b7fda1387b813710d1af6d6e29bb92186c8820", [:rebar3], [], "hexpm", "b8acbf89a224bc217c3070ca8bebc6eb236dbe7f9767993b274084ea044d35f0"}, - "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, "recon": {:hex, :recon, "2.5.4", "05dd52a119ee4059fa9daa1ab7ce81bc7a8161a2f12e9d42e9d551ffd2ba901c", [:mix, :rebar3], [], "hexpm", "e9ab01ac7fc8572e41eb59385efeb3fb0ff5bf02103816535bacaedf327d0263"}, "req": {:hex, :req, "0.3.12", "f84c2f9e7cc71c81d7cbeacf7c61e763e53ab5f3065703792a4ab264b4f22672", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "c91103d4d1c8edeba90c84e0ba223a59865b673eaab217bfd17da3aa54ab136c"}, "rustler": {:hex, :rustler, "0.29.1", "880f20ae3027bd7945def6cea767f5257bc926f33ff50c0d5d5a5315883c084d", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "109497d701861bfcd26eb8f5801fe327a8eef304f56a5b63ef61151ff44ac9b6"}, From bf0853baa9cbcb1abf6a5e096a857c00d420d54a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Thu, 4 Jul 2024 19:51:26 +0200 Subject: [PATCH 09/97] chore: new pgbench task (#381) --- Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Makefile b/Makefile index 7efcc144..5f243dc7 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,10 @@ db_rebuild: docker-compose -f ./docker-compose.db.yml build make db_start +PGBENCH_USER ?= postgres.sys +PGBENCH_PORT ?= 6543 +PGBENCH_RATE ?= 5000 + pgbench_init: PGPASSWORD=postgres pgbench -i -h 127.0.0.1 -p 6432 -U postgres -d postgres @@ -74,6 +78,9 @@ pgbench_short: pgbench_long: PGPASSWORD=postgres pgbench -M extended --transactions 100 --jobs 10 --client 60 -h localhost -p 7654 -U transaction.localhost postgres +pgbench: + PGPASSWORD="postgres" pgbench postgres://${PGBENCH_USER}@localhost:${PGBENCH_PORT}/postgres?sslmode=disable -Srn -T 60 -j 8 -c 1000 -P 10 -M extended --rate ${PGBENCH_RATE} + clean: rm -rf _build && rm -rf deps From bd80abcbe025eb6e67be040581b293d94d68d673 Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 8 Jul 2024 11:13:45 +0200 Subject: [PATCH 10/97] feat: remove partisan (#383) --- Makefile | 2 - config/runtime.exs | 20 +--------- config/test.exs | 26 ------------- lib/cluster/strategy/postgres.ex | 64 +------------------------------- lib/supavisor/client_handler.ex | 10 ++--- lib/supavisor/db_handler.ex | 12 +++--- mix.exs | 4 +- mix.lock | 5 --- test/support/cluster.ex | 3 -- 9 files changed, 16 insertions(+), 130 deletions(-) diff --git a/Makefile b/Makefile index 5f243dc7..a5cfb699 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,6 @@ dev.node2: CLUSTER_POSTGRES="true" \ PROXY_PORT_SESSION="5442" \ PROXY_PORT_TRANSACTION="6553" \ - PARTISAN_PEER_PORT="10201" \ ERL_AFLAGS="-kernel shell_history enabled" \ iex --name node2@127.0.0.1 --cookie cookie -S mix phx.server @@ -41,7 +40,6 @@ dev.node3: CLUSTER_POSTGRES="true" \ PROXY_PORT_SESSION="5443" \ PROXY_PORT_TRANSACTION="6554" \ - PARTISAN_PEER_PORT="10202" \ ERL_AFLAGS="-kernel shell_history enabled" \ iex --name node3@127.0.0.1 --cookie cookie -S mix phx.server diff --git a/config/runtime.exs b/config/runtime.exs index d5909ca5..57089160 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -85,8 +85,7 @@ topologies = config: [ url: System.get_env("DATABASE_URL", "ecto://postgres:postgres@localhost:6432/postgres"), heartbeat_interval: 5_000, - channel_name: "supavisor_#{region}_#{maj}_#{min}", - channel_name_partisan: "supavisor_partisan_#{region}_#{maj}_#{min}" + channel_name: "supavisor_#{region}_#{maj}_#{min}" ] ] @@ -174,23 +173,6 @@ if config_env() != :test do tag: "AES.GCM.V1", key: System.get_env("VAULT_ENC_KEY") } ] - - config :partisan, - # Which overlay to use - peer_service_manager: :partisan_pluggable_peer_service_manager, - listen_addrs: [ - { - System.get_env("PARTISAN_PEER_IP", "127.0.0.1"), - String.to_integer(System.get_env("PARTISAN_PEER_PORT", "20100")) - } - ], - channels: [ - data: %{parallelism: System.get_env("PARTISAN_PARALLELISM", "5") |> String.to_integer()} - ], - # Encoding for pid(), reference() and names - pid_encoding: false, - ref_encoding: false, - remote_ref_format: :improper_list end if System.get_env("LOGS_ENGINE") == "logflare" do diff --git a/config/test.exs b/config/test.exs index dcd59c57..13878d03 100644 --- a/config/test.exs +++ b/config/test.exs @@ -25,17 +25,6 @@ config :supavisor, Supavisor.Repo, pool_size: 10, port: 6432 -config :partisan, - # Which overlay to use - peer_service_manager: :partisan_pluggable_peer_service_manager, - # The listening port for Partisan TCP/IP connections - peer_port: 10200, - channels: [data: %{parallelism: 1}], - # Encoding for pid(), reference() and names - pid_encoding: false, - ref_encoding: false, - remote_ref_format: :improper_list - # We don't run a server during test. If one is required, # you can enable the server option below. config :supavisor, SupavisorWeb.Endpoint, @@ -58,18 +47,3 @@ config :logger, :console, # Initialize plugs at runtime for faster test compilation config :phoenix, :plug_init_mode, :runtime - -config :partisan, - peer_service_manager: :partisan_pluggable_peer_service_manager, - listen_addrs: [ - { - System.get_env("PARTISAN_PEER_IP", "127.0.0.1"), - String.to_integer(System.get_env("PARTISAN_PEER_PORT", "10200")) - } - ], - channels: [ - data: %{parallelism: System.get_env("PARTISAN_PARALLELISM", "5") |> String.to_integer()} - ], - pid_encoding: false, - ref_encoding: false, - remote_ref_format: :improper_list diff --git a/lib/cluster/strategy/postgres.ex b/lib/cluster/strategy/postgres.ex index 89493841..2020aacb 100644 --- a/lib/cluster/strategy/postgres.ex +++ b/lib/cluster/strategy/postgres.ex @@ -38,7 +38,6 @@ defmodule Cluster.Strategy.Postgres do state.config |> Keyword.put_new(:heartbeat_interval, 5_000) |> Keyword.put_new(:channel_name, "cluster") - |> Keyword.put_new(:channel_name_partisan, "cluster_partisan") |> Keyword.delete(:url) meta = %{ @@ -54,8 +53,7 @@ defmodule Cluster.Strategy.Postgres do def handle_continue(:connect, state) do with {:ok, conn} <- P.start_link(state.meta.opts.()), {:ok, conn_notif} <- P.Notifications.start_link(state.meta.opts.()), - {_, _} <- P.Notifications.listen(conn_notif, state.config[:channel_name]), - {_, _} <- P.Notifications.listen(conn_notif, state.config[:channel_name_partisan]) do + {_, _} <- P.Notifications.listen(conn_notif, state.config[:channel_name]) do Logger.info(state.topology, "Connected to Postgres database") meta = %{ @@ -76,24 +74,15 @@ defmodule Cluster.Strategy.Postgres do def handle_info(:heartbeat, state) do Process.cancel_timer(state.meta.heartbeat_ref) P.query(state.meta.conn, "NOTIFY #{state.config[:channel_name]}, '#{node()}'", []) - - P.query( - state.meta.conn, - "NOTIFY #{state.config[:channel_name_partisan]}, '#{partisan_peer_spec_enc()}'", - [] - ) - ref = heartbeat(state.config[:heartbeat_interval]) {:noreply, put_in(state.meta.heartbeat_ref, ref)} end def handle_info({:notification, _, _, channel, msg}, state) do disterl = state.config[:channel_name] - partisan = state.config[:channel_name_partisan] case channel do ^disterl -> handle_channels(:disterl, msg, state) - ^partisan -> handle_channels(:partisan, msg, state) other -> Logger.error(state.topology, "Unknown channel: #{other}") end @@ -105,20 +94,6 @@ defmodule Cluster.Strategy.Postgres do {:noreply, state} end - def code_change("1.1.48", state, _) do - Logger.info(state.topology, "Update state from 1.1.48") - - partisan_channel = - Application.get_env(:libcluster, :topologies) - |> get_in([:postgres, :config, :channel_name_partisan]) - - new_config = - state.config - |> Keyword.put(:channel_name_partisan, partisan_channel) - - {:ok, %{state | config: new_config}} - end - def code_change(_, state, _), do: {:ok, state} ### Internal functions @@ -127,7 +102,7 @@ defmodule Cluster.Strategy.Postgres do Process.send_after(self(), :heartbeat, interval) end - @spec handle_channels(:disterl | :partisan, String.t(), map()) :: any() + @spec handle_channels(:disterl, String.t(), map()) :: any() def handle_channels(:disterl, msg, state) do node = String.to_atom(msg) @@ -144,39 +119,4 @@ defmodule Cluster.Strategy.Postgres do end end end - - def handle_channels(:partisan, msg, state) do - spec = partisan_peer_spec_dec(msg) - - if spec.name not in [:partisan.node() | :partisan.nodes()] do - spec = partisan_peer_spec_dec(msg) - topology = state.topology - - Logger.debug( - topology, - "Trying to connect to partisan node: #{inspect(spec, pretty: true)}" - ) - - case :partisan_peer_service.join(spec) do - :ok -> - Logger.debug(topology, "Connected to node: #{inspect(spec, pretty: true)}") - - other -> - Logger.error(topology, "Failed to connect to partisan node: #{other}") - end - end - end - - @spec partisan_peer_spec_enc() :: String.t() - def partisan_peer_spec_enc() do - :partisan.node_spec() - |> :erlang.term_to_binary() - |> Base.encode64() - end - - @spec partisan_peer_spec_dec(String.t()) :: term() - def partisan_peer_spec_dec(spec) do - Base.decode64!(spec) - |> :erlang.binary_to_term() - end end diff --git a/lib/supavisor/client_handler.ex b/lib/supavisor/client_handler.ex index 96ac9420..a4afbc90 100644 --- a/lib/supavisor/client_handler.ex +++ b/lib/supavisor/client_handler.ex @@ -1,7 +1,7 @@ defmodule Supavisor.ClientHandler do @moduledoc """ This module is responsible for handling incoming connections to the Supavisor server. It is - implemented as a Ranch protocol behavior and a partisan_gen_statem behavior. It handles SSL negotiation, + implemented as a Ranch protocol behavior and a gen_statem behavior. It handles SSL negotiation, user authentication, tenant subscription, and dispatching of messages to the appropriate tenant supervisor. Each client connection is assigned to a specific tenant supervisor. """ @@ -9,7 +9,7 @@ defmodule Supavisor.ClientHandler do require Logger @behaviour :ranch_protocol - @behaviour :partisan_gen_statem + @behaviour :gen_statem alias Supavisor, as: S alias Supavisor.DbHandler, as: Db @@ -27,12 +27,12 @@ defmodule Supavisor.ClientHandler do def callback_mode, do: [:handle_event_function] def client_cast(pid, bin, status) do - :partisan_gen_statem.cast(pid, {:client_cast, bin, status}) + :gen_statem.cast(pid, {:client_cast, bin, status}) end @spec client_call(pid, iodata(), atom()) :: :ok | {:error, term()} def client_call(pid, bin, status), - do: :partisan_gen_statem.call(pid, {:client_call, bin, status}, 30_000) + do: :gen_statem.call(pid, {:client_call, bin, status}, 30_000) @impl true def init(_), do: :ignore @@ -70,7 +70,7 @@ defmodule Supavisor.ClientHandler do log_level: nil } - :partisan_gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) + :gen_statem.enter_loop(__MODULE__, [hibernate_after: 5_000], :exchange, data) end @impl true diff --git a/lib/supavisor/db_handler.ex b/lib/supavisor/db_handler.ex index acdb1141..2c105c3d 100644 --- a/lib/supavisor/db_handler.ex +++ b/lib/supavisor/db_handler.ex @@ -6,7 +6,7 @@ defmodule Supavisor.DbHandler do require Logger - @behaviour :partisan_gen_statem + @behaviour :gen_statem alias Supavisor, as: S alias Supavisor.ClientHandler, as: Client @@ -22,26 +22,26 @@ defmodule Supavisor.DbHandler do @async_send_limit 1_000 def start_link(config) do - :partisan_gen_statem.start_link(__MODULE__, config, hibernate_after: 5_000) + :gen_statem.start_link(__MODULE__, config, hibernate_after: 5_000) end @spec call(pid(), pid(), binary()) :: :ok | {:error, any()} | {:buffering, non_neg_integer()} - def call(pid, caller, msg), do: :partisan_gen_statem.call(pid, {:db_call, caller, msg}, 15_000) + def call(pid, caller, msg), do: :gen_statem.call(pid, {:db_call, caller, msg}, 15_000) @spec cast(pid(), pid(), binary()) :: :ok | {:error, any()} | {:buffering, non_neg_integer()} - def cast(pid, caller, msg), do: :partisan_gen_statem.cast(pid, {:db_cast, caller, msg}) + def cast(pid, caller, msg), do: :gen_statem.cast(pid, {:db_cast, caller, msg}) @spec get_state_and_mode(pid()) :: {:ok, {state, Supavisor.mode()}} | {:error, term()} def get_state_and_mode(pid) do try do - {:ok, :partisan_gen_statem.call(pid, :get_state_and_mode, 5_000)} + {:ok, :gen_statem.call(pid, :get_state_and_mode, 5_000)} catch error, reason -> {:error, {error, reason}} end end @spec stop(pid()) :: :ok - def stop(pid), do: :partisan_gen_statem.stop(pid, :client_termination, 5_000) + def stop(pid), do: :gen_statem.stop(pid, :client_termination, 5_000) @impl true def init(args) do diff --git a/mix.exs b/mix.exs index 6e1dca6b..675fe407 100644 --- a/mix.exs +++ b/mix.exs @@ -22,11 +22,12 @@ defmodule Supavisor.MixProject do [ mod: {Supavisor.Application, []}, extra_applications: - [:logger, :runtime_tools, :os_mon, :ssl, :partisan] ++ extra_applications(Mix.env()) + [:logger, :runtime_tools, :os_mon, :ssl] ++ extra_applications(Mix.env()) ] end defp extra_applications(:test), do: [:common_test] + defp extra_applications(:dev), do: [:wx, :observer] defp extra_applications(_), do: [] # Specifies which paths to compile per environment. @@ -71,7 +72,6 @@ defmodule Supavisor.MixProject do # pooller # {:poolboy, "~> 1.5.2"}, {:poolboy, git: "https://github.com/abc3/poolboy.git", tag: "v0.0.2"}, - {:partisan, git: "https://github.com/lasp-lang/partisan.git", tag: "v5.0.0-rc.12"}, {:syn, "~> 3.3"}, {:pgo, "~> 0.13"}, {:rustler, "~> 0.29.1"}, diff --git a/mix.lock b/mix.lock index e26edd8e..bf50709b 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,4 @@ %{ - "acceptor_pool": {:hex, :acceptor_pool, "1.0.0", "43c20d2acae35f0c2bcd64f9d2bde267e459f0f3fd23dab26485bf518c281b21", [:rebar3], [], "hexpm", "0cbcd83fdc8b9ad2eee2067ef8b91a14858a5883cb7cd800e6fcd5803e158788"}, "artificery": {:hex, :artificery, "0.4.3", "0bc4260f988dcb9dda4b23f9fc3c6c8b99a6220a331534fdf5bf2fd0d4333b02", [:mix], [], "hexpm", "12e95333a30e20884e937abdbefa3e7f5e05609c2ba8cf37b33f000b9ffc0504"}, "backoff": {:hex, :backoff, "1.1.6", "83b72ed2108ba1ee8f7d1c22e0b4a00cfe3593a67dbc792799e8cce9f42f796b", [:rebar3], [], "hexpm", "cf0cfff8995fb20562f822e5cc47d8ccf664c5ecdc26a684cbe85c225f9d7c39"}, "benchee": {:hex, :benchee, "1.1.0", "f3a43817209a92a1fade36ef36b86e1052627fd8934a8b937ac9ab3a76c43062", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}], "hexpm", "7da57d545003165a012b587077f6ba90b89210fd88074ce3c60ce239eb5e6d93"}, @@ -46,7 +45,6 @@ "open_api_spex": {:hex, :open_api_spex, "3.18.0", "f9952b6bc8a1bf14168f3754981b7c8d72d015112bfedf2588471dd602e1e715", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "37849887ab67efab052376401fac28c0974b273ffaecd98f4532455ca0886464"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.2.1", "7b69ed4f40025c005de0b74fce8c0549625d59cb4df12d15c32fe6dc5076ff42", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "6d7a27b7cad2ad69a09cabf6670514cafcec717c8441beb5c96322bac3d05350"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, - "partisan": {:git, "https://github.com/lasp-lang/partisan.git", "a6a59b3ed406037099773861dbaad4e97b5b9142", [tag: "v5.0.0-rc.12"]}, "pg_types": {:hex, :pg_types, "0.4.0", "3ce365c92903c5bb59c0d56382d842c8c610c1b6f165e20c4b652c96fa7e9c14", [:rebar3], [], "hexpm", "b02efa785caececf9702c681c80a9ca12a39f9161a846ce17b01fb20aeeed7eb"}, "pgo": {:hex, :pgo, "0.14.0", "f53711d103d7565db6fc6061fcf4ff1007ab39892439be1bb02d9f686d7e6663", [:rebar3], [{:backoff, "~> 1.1.6", [hex: :backoff, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:pg_types, "~> 0.4.0", [hex: :pg_types, repo: "hexpm", optional: false]}], "hexpm", "71016c22599936e042dc0012ee4589d24c71427d266292f775ebf201d97df9c9"}, "phoenix": {:hex, :phoenix, "1.7.10", "02189140a61b2ce85bb633a9b6fd02dff705a5f1596869547aeb2b2b95edd729", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "cf784932e010fd736d656d7fead6a584a4498efefe5b8227e9f383bf15bb79d0"}, @@ -64,7 +62,6 @@ "poolboy": {:git, "https://github.com/abc3/poolboy.git", "999ec7f5c7282d515020bb058b4832029d6d07bc", [tag: "v0.0.2"]}, "postgrex": {:hex, :postgrex, "0.17.3", "c92cda8de2033a7585dae8c61b1d420a1a1322421df84da9a82a6764580c503d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "946cf46935a4fdca7a81448be76ba3503cff082df42c6ec1ff16a4bdfbfb098d"}, "prom_ex": {:hex, :prom_ex, "1.8.0", "662615e1d2f2ab3e0dc13a51c92ad0ccfcab24336a90cb9b114ee1bce9ef88aa", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.15", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.3", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "3eea763dfa941e25de50decbf17a6a94dbd2270e7b32f88279aa6e9bbb8e23e7"}, - "quickrand": {:hex, :quickrand, "2.0.7", "d2bd76676a446e6a058d678444b7fda1387b813710d1af6d6e29bb92186c8820", [:rebar3], [], "hexpm", "b8acbf89a224bc217c3070ca8bebc6eb236dbe7f9767993b274084ea044d35f0"}, "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, "recon": {:hex, :recon, "2.5.4", "05dd52a119ee4059fa9daa1ab7ce81bc7a8161a2f12e9d42e9d551ffd2ba901c", [:mix, :rebar3], [], "hexpm", "e9ab01ac7fc8572e41eb59385efeb3fb0ff5bf02103816535bacaedf327d0263"}, "req": {:hex, :req, "0.3.12", "f84c2f9e7cc71c81d7cbeacf7c61e763e53ab5f3065703792a4ab264b4f22672", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "c91103d4d1c8edeba90c84e0ba223a59865b673eaab217bfd17da3aa54ab136c"}, @@ -79,9 +76,7 @@ "tesla": {:hex, :tesla, "1.8.0", "d511a4f5c5e42538d97eef7c40ec4f3e44effdc5068206f42ed859e09e51d1fd", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "10501f360cd926a309501287470372af1a6e1cbed0f43949203a4c13300bc79f"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, - "types": {:hex, :types, "0.1.8", "5782b67231e8c174fe2835395e71e669fe0121076779d2a09f1c0d58ee0e2f13", [:rebar3], [], "hexpm", "04285239f4954c5ede56f78ed7778ede24e3f2e997f7b16402a167af0cc2658a"}, "unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"}, - "uuid": {:hex, :uuid_erl, "2.0.5", "60faeeb7edfd40847ed13cb0dd1044baabe4e79a00c0ca9c4d13a073914b1016", [:rebar3], [{:quickrand, ">= 2.0.5", [hex: :quickrand, repo: "hexpm", optional: false]}], "hexpm", "e54373262ca88401689277947c54b95e9ecbc977bd5c57c9dd44ad9da278e360"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.5", "9dfeee8269b27e958a65b3e235b7e447769f66b5b5925385f5a569269164a210", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b977ba4a01918acbf77045ff88de7f6972c2a009213c515a445c48f224ffce9"}, } diff --git a/test/support/cluster.ex b/test/support/cluster.ex index 9243eaf6..6847111a 100644 --- a/test/support/cluster.ex +++ b/test/support/cluster.ex @@ -20,9 +20,6 @@ defmodule Supavisor.Support.Cluster do {:supavisor, :region} -> "usa" - {:partisan, :listen_addrs} -> - [{"127.0.0.1", 10201}] - _ -> val end From 9b0ce29609191d212658aff9c787be551e199379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Mon, 8 Jul 2024 11:14:24 +0200 Subject: [PATCH 11/97] chore: update tests to use peer module instead of ct_slave (#368) --- lib/supavisor/application.ex | 30 +++++++++++-------- test/integration/proxy_test.exs | 11 ++++--- test/supavisor/prom_ex_test.exs | 6 ++-- test/supavisor/syn_handler_test.exs | 6 ++-- .../controllers/metrics_controller_test.exs | 4 +++ test/support/cluster.ex | 24 +++++++++++++-- test/test_helper.exs | 7 ----- 7 files changed, 55 insertions(+), 33 deletions(-) diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index 0a90c5e5..11ed773c 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -34,19 +34,23 @@ defmodule Supavisor.Application do ] for {key, port, mode} <- proxy_ports do - :ranch.start_listener( - key, - :ranch_tcp, - %{ - max_connections: String.to_integer(System.get_env("MAX_CONNECTIONS") || "25000"), - num_acceptors: String.to_integer(System.get_env("NUM_ACCEPTORS") || "100"), - socket_opts: [port: port, keepalive: true] - }, - Supavisor.ClientHandler, - %{mode: mode} - ) - |> then(&"Proxy started #{mode} on port #{port}, result: #{inspect(&1)}") - |> Logger.warning() + case :ranch.start_listener( + key, + :ranch_tcp, + %{ + max_connections: String.to_integer(System.get_env("MAX_CONNECTIONS") || "25000"), + num_acceptors: String.to_integer(System.get_env("NUM_ACCEPTORS") || "100"), + socket_opts: [inet_backend: :socket, port: port, keepalive: true] + }, + Supavisor.ClientHandler, + %{mode: mode} + ) do + {:ok, _pid} -> + Logger.notice("Proxy started #{mode} on port #{port}") + + error -> + Logger.error("Proxy on #{port} not started because of #{inspect(error)}") + end end :syn.set_event_handler(Supavisor.SynHandler) diff --git a/test/integration/proxy_test.exs b/test/integration/proxy_test.exs index a3720f2b..aaee5ff7 100644 --- a/test/integration/proxy_test.exs +++ b/test/integration/proxy_test.exs @@ -75,6 +75,8 @@ defmodule Supavisor.Integration.ProxyTest do end test "query via another node", %{proxy: proxy, user: user} do + {:ok, _pid, node2} = Supavisor.Support.Cluster.start_node() + sup = Enum.reduce_while(1..30, nil, fn _, acc -> case Supavisor.get_global_sup({@tenant, user, :transaction}) do @@ -89,7 +91,7 @@ defmodule Supavisor.Integration.ProxyTest do assert sup == :erpc.call( - :"secondary@127.0.0.1", + node2, Supavisor, :get_global_sup, [{@tenant, user, :transaction}], @@ -114,7 +116,7 @@ defmodule Supavisor.Integration.ProxyTest do assert sup == :erpc.call( - :"secondary@127.0.0.1", + node2, Supavisor, :get_global_sup, [{@tenant, user, :transaction}], @@ -188,10 +190,11 @@ defmodule Supavisor.Integration.ProxyTest do [{_, client_pid, _}] = Supavisor.get_local_manager({{:single, @tenant}, "transaction", :transaction, "postgres"}) |> :sys.get_state() - |> then(& &1[:tid]) + |> Access.get(:tid) |> :ets.tab2list() - {state, %{db_pid: db_pid}} = :sys.get_state(client_pid) + assert {state, map} = :sys.get_state(client_pid) + assert %{db_pid: db_pid} = map assert {:idle, nil} = {state, db_pid} :gen_statem.stop(pid) diff --git a/test/supavisor/prom_ex_test.exs b/test/supavisor/prom_ex_test.exs index d29dc23c..246f1417 100644 --- a/test/supavisor/prom_ex_test.exs +++ b/test/supavisor/prom_ex_test.exs @@ -28,10 +28,10 @@ defmodule Supavisor.PromExTest do test "remove tenant tag upon termination", %{proxy: proxy, user: user, db_name: db_name} do assert PromEx.get_metrics() =~ "tenant=\"#{@tenant}\"" - GenServer.stop(proxy) - Supavisor.stop({{:single, @tenant}, user, :transaction, db_name}) + :ok = GenServer.stop(proxy) + :ok = Supavisor.stop({{:single, @tenant}, user, :transaction, db_name}) - Process.sleep(500) + Process.sleep(1000) refute PromEx.get_metrics() =~ "tenant=\"#{@tenant}\"" end diff --git a/test/supavisor/syn_handler_test.exs b/test/supavisor/syn_handler_test.exs index edfa4f5a..2404e2c1 100644 --- a/test/supavisor/syn_handler_test.exs +++ b/test/supavisor/syn_handler_test.exs @@ -7,7 +7,7 @@ defmodule Supavisor.SynHandlerTest do @id {{:single, "syn_tenant"}, "postgres", :session, "postgres"} test "resolving conflict" do - node2 = :"secondary@127.0.0.1" + {:ok, _pid, node2} = Supavisor.Support.Cluster.start_node() secret = %{alias: "postgres"} auth_secret = {:password, fn -> secret end} @@ -16,7 +16,7 @@ defmodule Supavisor.SynHandlerTest do assert pid2 == Supavisor.get_global_sup(@id) assert node(pid2) == node2 true = Node.disconnect(node2) - Process.sleep(500) + Process.sleep(1000) assert nil == Supavisor.get_global_sup(@id) {:ok, pid1} = Supavisor.start(@id, auth_secret) @@ -28,7 +28,7 @@ defmodule Supavisor.SynHandlerTest do msg = "Resolving syn_tenant conflict, stop local pid" - assert capture_log(fn -> Logger.warn(msg) end) =~ + assert capture_log(fn -> Logger.warning(msg) end) =~ msg assert pid2 == Supavisor.get_global_sup(@id) diff --git a/test/supavisor_web/controllers/metrics_controller_test.exs b/test/supavisor_web/controllers/metrics_controller_test.exs index 5bbd3407..9bb0963f 100644 --- a/test/supavisor_web/controllers/metrics_controller_test.exs +++ b/test/supavisor_web/controllers/metrics_controller_test.exs @@ -13,6 +13,10 @@ defmodule SupavisorWeb.MetricsControllerTest do end test "exporting metrics", %{conn: conn} do + {:ok, _pid, node2} = Supavisor.Support.Cluster.start_node() + + Node.connect(node2) + :meck.expect(Supavisor.Jwt, :authorize, fn _token, _secret -> {:ok, %{}} end) conn = get(conn, Routes.metrics_path(conn, :index)) assert conn.status == 200 diff --git a/test/support/cluster.ex b/test/support/cluster.ex index 6847111a..921bf1f2 100644 --- a/test/support/cluster.ex +++ b/test/support/cluster.ex @@ -3,7 +3,24 @@ defmodule Supavisor.Support.Cluster do This module provides functionality to help handle distributive mode for testing. """ - def apply_config(node) do + def start_node(name \\ :peer.random_name()) do + {:ok, pid, node} = + :peer.start_link(%{ + name: name, + host: ~c"127.0.0.1", + longnames: true, + connection: :standard_io + }) + + :peer.call(pid, :logger, :set_primary_config, [:level, :none]) + true = :peer.call(pid, :code, :set_path, [:code.get_path()]) + apply_config(pid) + :peer.call(pid, Application, :ensure_all_started, [:supavisor]) + + {:ok, pid, node} + end + + defp apply_config(pid) do for {app_name, _, _} <- Application.loaded_applications() do for {key, val} <- Application.get_all_env(app_name) do val = @@ -24,9 +41,10 @@ defmodule Supavisor.Support.Cluster do val end - :rpc.call(node, Application, :put_env, [app_name, key, val, [persistent: true]]) - :rpc.call(node, Supavisor.Monitoring.PromEx, :set_metrics_tags, []) + :peer.call(pid, Application, :put_env, [app_name, key, val]) end end + + :peer.call(pid, Supavisor.Monitoring.PromEx, :set_metrics_tags, []) end end diff --git a/test/test_helper.exs b/test/test_helper.exs index a3a81c8b..92ecaa63 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,11 +1,4 @@ {:ok, _} = Node.start(:"primary@127.0.0.1", :longnames) -node2 = :"secondary@127.0.0.1" -:ct_slave.start(node2) -true = :erpc.call(node2, :code, :set_path, [:code.get_path()]) - -Supavisor.Support.Cluster.apply_config(node2) - -{:ok, _} = :erpc.call(node2, :application, :ensure_all_started, [:supavisor]) Cachex.start_link(name: Supavisor.Cache) From ddf8f0194a1853e2c38ed562aa65ca776cce9e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Mon, 8 Jul 2024 11:23:16 +0200 Subject: [PATCH 12/97] ft: use Peep as a PromEx storage (#384) --- Makefile | 11 +++- config/dev.exs | 9 ++- lib/supavisor/monitoring/prom_ex.ex | 7 ++- lib/supavisor/monitoring/telem.ex | 14 ++--- lib/supavisor/monitoring/tenant.ex | 91 ++++++++++++++++++----------- mix.exs | 3 +- mix.lock | 46 ++++++++------- 7 files changed, 110 insertions(+), 71 deletions(-) diff --git a/Makefile b/Makefile index a5cfb699..8dbbf35d 100644 --- a/Makefile +++ b/Makefile @@ -66,18 +66,25 @@ db_rebuild: PGBENCH_USER ?= postgres.sys PGBENCH_PORT ?= 6543 PGBENCH_RATE ?= 5000 +PGBENCH_DURATION ?= 60 +PGBENCH_CLIENTS ?= 1000 pgbench_init: PGPASSWORD=postgres pgbench -i -h 127.0.0.1 -p 6432 -U postgres -d postgres pgbench_short: - PGPASSWORD=postgres pgbench -M extended --transactions 5 --jobs 4 --client 1 -h localhost -p 7654 -U transaction.localhost postgres + PGPASSWORD=postgres pgbench -M extended --transactions 5 --jobs 4 --client 1 -h localhost -p 6543 -U postgres.sys postgres pgbench_long: PGPASSWORD=postgres pgbench -M extended --transactions 100 --jobs 10 --client 60 -h localhost -p 7654 -U transaction.localhost postgres pgbench: - PGPASSWORD="postgres" pgbench postgres://${PGBENCH_USER}@localhost:${PGBENCH_PORT}/postgres?sslmode=disable -Srn -T 60 -j 8 -c 1000 -P 10 -M extended --rate ${PGBENCH_RATE} + PGPASSWORD="postgres" pgbench \ + postgres://${PGBENCH_USER}@localhost:${PGBENCH_PORT}/postgres?sslmode=disable \ + -Srn -T ${PGBENCH_DURATION} \ + -j 8 -c ${PGBENCH_CLIENTS} \ + -P 10 -M extended \ + --rate ${PGBENCH_RATE} clean: rm -rf _build && rm -rf deps diff --git a/config/dev.exs b/config/dev.exs index 843b9f47..b941e7e8 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -58,11 +58,16 @@ config :supavisor, SupavisorWeb.Endpoint, ] ] +config :logger, + compile_time_purge_matching: [ + [level_lower_than: :info] + ] + # Configures Elixir's Logger config :logger, :console, format: "$time [$level] $message $metadata\n", - level: :debug, - # level: :error, + # level: :debug, + level: :notice, metadata: [:error_code, :file, :line, :pid, :project, :user, :mode, :type] # Set a higher stacktrace during development. Avoid configuring such diff --git a/lib/supavisor/monitoring/prom_ex.ex b/lib/supavisor/monitoring/prom_ex.ex index 96a0eff3..d62788ca 100644 --- a/lib/supavisor/monitoring/prom_ex.ex +++ b/lib/supavisor/monitoring/prom_ex.ex @@ -5,7 +5,7 @@ defmodule Supavisor.Monitoring.PromEx do and provides a function to remove remote metrics associated with a specific tenant. """ - use PromEx, otp_app: :supavisor + use PromEx, otp_app: :supavisor, store: PromEx.Storage.Peep require Logger alias PromEx.Plugins @@ -34,7 +34,10 @@ defmodule Supavisor.Monitoring.PromEx do meta = %{tenant: tenant, user: user, mode: mode, type: type, db_name: db_name} Supavisor.Monitoring.PromEx.Metrics - |> :ets.select_delete([{{{:_, meta}, :_}, [], [true]}]) + |> :ets.select_delete([ + {{{:_, meta}, :_}, [], [true]}, + {{{:_, meta, :_}, :_}, [], [true]} + ]) end @spec set_metrics_tags() :: map() diff --git a/lib/supavisor/monitoring/telem.ex b/lib/supavisor/monitoring/telem.ex index 460b7524..057e64c2 100644 --- a/lib/supavisor/monitoring/telem.ex +++ b/lib/supavisor/monitoring/telem.ex @@ -6,29 +6,29 @@ defmodule Supavisor.Monitoring.Telem do alias Supavisor, as: S @spec network_usage(:client | :db, S.sock(), S.id(), map()) :: {:ok | :error, map()} - def network_usage(type, {mod, socket}, id, stats) do + def network_usage(type, {mod, socket}, id, _stats) do mod = if mod == :ssl, do: :ssl, else: :inet case mod.getstat(socket, [:recv_oct, :send_oct]) do {:ok, [{:recv_oct, recv_oct}, {:send_oct, send_oct}]} -> - diff = %{ - send_oct: send_oct - Map.get(stats, :send_oct, 0), - recv_oct: recv_oct - Map.get(stats, :recv_oct, 0) + stats = %{ + send_oct: send_oct, + recv_oct: recv_oct } {{ptype, tenant}, user, mode, db_name} = id :telemetry.execute( [:supavisor, type, :network, :stat], - diff, + stats, %{tenant: tenant, user: user, mode: mode, type: ptype, db_name: db_name} ) - {:ok, %{recv_oct: recv_oct, send_oct: send_oct}} + {:ok, %{}} {:error, reason} -> Logger.error("Failed to get socket stats: #{inspect(reason)}") - {:error, stats} + {:error, %{}} end end diff --git a/lib/supavisor/monitoring/tenant.ex b/lib/supavisor/monitoring/tenant.ex index ff2309bc..55c12799 100644 --- a/lib/supavisor/monitoring/tenant.ex +++ b/lib/supavisor/monitoring/tenant.ex @@ -26,7 +26,12 @@ defmodule Supavisor.PromEx.Plugins.Tenant do ] end - def client_metrics() do + defmodule Buckets do + use Peep.Buckets.Custom, + buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000] + end + + defp client_metrics() do Event.build( :supavisor_tenant_client_event_metrics, [ @@ -36,9 +41,10 @@ defmodule Supavisor.PromEx.Plugins.Tenant do measurement: :duration, description: "Duration of the checkout local process in the tenant db pool.", tags: @tags, - unit: {:microsecond, :millisecond}, + unit: {:native, :millisecond}, reporter_options: [ - buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000] + buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000], + peep_bucket_calculator: Buckets ] ), distribution( @@ -47,9 +53,10 @@ defmodule Supavisor.PromEx.Plugins.Tenant do measurement: :duration, description: "Duration of the checkout remote process in the tenant db pool.", tags: @tags, - unit: {:microsecond, :millisecond}, + unit: {:native, :millisecond}, reporter_options: [ - buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000] + buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000], + peep_bucket_calculator: Buckets ] ), distribution( @@ -60,7 +67,8 @@ defmodule Supavisor.PromEx.Plugins.Tenant do tags: @tags, unit: {:native, :millisecond}, reporter_options: [ - buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000] + buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000], + peep_bucket_calculator: Buckets ] ), distribution( @@ -71,22 +79,29 @@ defmodule Supavisor.PromEx.Plugins.Tenant do tags: @tags, unit: {:native, :millisecond}, reporter_options: [ - buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000] + buckets: [1, 5, 10, 100, 1_000, 5_000, 10_000], + peep_bucket_calculator: Buckets ] ), - sum( + last_value( [:supavisor, :client, :network, :recv], event_name: [:supavisor, :client, :network, :stat], measurement: :recv_oct, description: "The total number of bytes received by clients.", - tags: @tags + tags: @tags, + reporter_options: [ + prometheus_type: :sum + ] ), - sum( + last_value( [:supavisor, :client, :network, :send], event_name: [:supavisor, :client, :network, :stat], measurement: :send_oct, description: "The total number of bytes sent by clients.", - tags: @tags + tags: @tags, + reporter_options: [ + prometheus_type: :sum + ] ), counter( [:supavisor, :client, :queries, :count], @@ -117,52 +132,58 @@ defmodule Supavisor.PromEx.Plugins.Tenant do event_name: [:supavisor, :client_handler, :stopped, :all], description: "The total number of stopped client_handler.", tags: @tags - ), - counter( - [:supavisor, :db_handler, :started, :count], - event_name: [:supavisor, :db_handler, :started, :all], - description: "The total number of created db_handler.", - tags: @tags - ), - counter( - [:supavisor, :db_handler, :stopped, :count], - event_name: [:supavisor, :db_handler, :stopped, :all], - description: "The total number of stopped db_handler.", - tags: @tags - ), - counter( - [:supavisor, :db_handler, :db_connection, :count], - event_name: [:supavisor, :db_handler, :db_connection, :all], - description: "The total number of database connections by db_handler.", - tags: @tags ) ] ) end - def db_metrics() do + defp db_metrics() do Event.build( :supavisor_tenant_db_event_metrics, [ - sum( + last_value( [:supavisor, :db, :network, :recv], event_name: [:supavisor, :db, :network, :stat], measurement: :recv_oct, description: "The total number of bytes received by db process", - tags: @tags + tags: @tags, + reporter_options: [ + prometheus_type: :sum + ] ), - sum( + last_value( [:supavisor, :db, :network, :send], event_name: [:supavisor, :db, :network, :stat], measurement: :send_oct, description: "The total number of bytes sent by db process", + tags: @tags, + reporter_options: [ + prometheus_type: :sum + ] + ), + counter( + [:supavisor, :db_handler, :started, :count], + event_name: [:supavisor, :db_handler, :started, :all], + description: "The total number of created db_handler.", + tags: @tags + ), + counter( + [:supavisor, :db_handler, :stopped, :count], + event_name: [:supavisor, :db_handler, :stopped, :all], + description: "The total number of stopped db_handler.", + tags: @tags + ), + counter( + [:supavisor, :db_handler, :db_connection, :count], + event_name: [:supavisor, :db_handler, :db_connection, :all], + description: "The total number of database connections by db_handler.", tags: @tags ) ] ) end - def concurrent_connections(poll_rate) do + defp concurrent_connections(poll_rate) do Polling.build( :supavisor_concurrent_connections, poll_rate, @@ -194,7 +215,7 @@ defmodule Supavisor.PromEx.Plugins.Tenant do ) end - def concurrent_tenants(poll_rate) do + defp concurrent_tenants(poll_rate) do Polling.build( :supavisor_concurrent_tenants, poll_rate, diff --git a/mix.exs b/mix.exs index 675fe407..8d2f09e5 100644 --- a/mix.exs +++ b/mix.exs @@ -50,6 +50,7 @@ defmodule Supavisor.MixProject do {:phoenix_live_dashboard, "~> 0.7"}, {:telemetry_metrics, "~> 0.6"}, {:telemetry_poller, "~> 1.0"}, + {:peep, github: "hauleth/peep", branch: "ft/custom-prometheus-types", override: true}, {:jason, "~> 1.2"}, {:plug_cowboy, "~> 2.5"}, {:joken, "~> 2.5.0"}, @@ -58,7 +59,7 @@ defmodule Supavisor.MixProject do {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, {:benchee, "~> 1.1.0", only: :dev}, - {:prom_ex, "~> 1.8.0"}, + {:prom_ex, github: "hauleth/prom_ex", branch: "ft/add-peep-storage"}, {:open_api_spex, "~> 3.16"}, {:burrito, github: "burrito-elixir/burrito"}, {:libcluster, "~> 3.3.1"}, diff --git a/mix.lock b/mix.lock index bf50709b..a4125f3e 100644 --- a/mix.lock +++ b/mix.lock @@ -6,12 +6,12 @@ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "burrito": {:git, "https://github.com/burrito-elixir/burrito.git", "a60c6ab21156fc4c788907d33bfd0c546a022272", []}, "cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"}, - "castore": {:hex, :castore, "1.0.4", "ff4d0fb2e6411c0479b1d965a814ea6d00e51eb2f58697446e9c41a97d940b28", [:mix], [], "hexpm", "9418c1b8144e11656f0be99943db4caf04612e3eaecefb5dae9a2a87565584f8"}, + "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, "cloak": {:hex, :cloak, "1.1.2", "7e0006c2b0b98d976d4f559080fabefd81f0e0a50a3c4b621f85ceeb563e80bb", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "940d5ac4fcd51b252930fd112e319ea5ae6ab540b722f3ca60a85666759b9585"}, "cloak_ecto": {:hex, :cloak_ecto, "1.2.0", "e86a3df3bf0dc8980f70406bcb0af2858bac247d55494d40bc58a152590bd402", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "8bcc677185c813fe64b786618bd6689b1707b35cd95acaae0834557b15a0c62f"}, - "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, + "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, - "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, + "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, @@ -24,10 +24,10 @@ "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, - "finch": {:hex, :finch, "0.16.0", "40733f02c89f94a112518071c0a91fe86069560f5dbdb39f9150042f44dcfb1a", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f660174c4d519e5fec629016054d60edd822cdfe2b7270836739ac2f97735ec5"}, - "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, + "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, + "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"}, "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"}, - "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, + "jason": {:hex, :jason, "1.4.3", "d3f984eeb96fe53b85d20e0b049f03e57d075b5acda3ac8d465c969a2536c17b", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "9a90e868927f7c777689baa16d86f4d0e086d968db5c05d917ccff6d443e58a3"}, "joken": {:hex, :joken, "2.5.0", "09be497d804b8115eb6f07615cef2e60c2a1008fb89dc0aef0d4c4b4609b99aa", [:mix], [{:jose, "~> 1.11.2", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "22b25c89617c5ed8ca7b31026340a25ea0f9ca7160f9706b79be9ed81fdf74e7"}, "jose": {:hex, :jose, "1.11.6", "613fda82552128aa6fb804682e3a616f4bc15565a048dabd05b1ebd5827ed965", [:mix, :rebar3], [], "hexpm", "6275cb75504f9c1e60eeacb771adfeee4905a9e182103aa59b53fed651ff9738"}, "jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"}, @@ -36,47 +36,49 @@ "logflare_etso": {:hex, :logflare_etso, "1.1.2", "040bd3e482aaf0ed20080743b7562242ec5079fd88a6f9c8ce5d8298818292e9", [:mix], [{:ecto, "~> 3.8", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "ab96be42900730a49b132891f43a9be1d52e4ad3ee9ed9cb92565c5f87345117"}, "logflare_logger_backend": {:git, "https://github.com/Logflare/logflare_logger_backend.git", "7fcc9f32ec48f466ddc1738709d7dc646cfc1e3a", [tag: "v0.11.4"]}, "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, - "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, - "mint": {:hex, :mint, "1.5.1", "8db5239e56738552d85af398798c80648db0e90f343c8469f6c6d8898944fb6f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4a63e1e76a7c3956abd2c72f370a0d0aecddc3976dea5c27eccbecfa5e7d5b1e"}, - "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, - "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, + "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, + "mint": {:hex, :mint, "1.6.2", "af6d97a4051eee4f05b5500671d47c3a67dac7386045d87a904126fd4bbcea2e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5ee441dffc1892f1ae59127f74afe8fd82fda6587794278d924e4d90ea3d63f9"}, + "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "observer_cli": {:hex, :observer_cli, "1.7.4", "3c1bfb6d91bf68f6a3d15f46ae20da0f7740d363ee5bc041191ce8722a6c4fae", [:mix, :rebar3], [{:recon, "~> 2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "50de6d95d814f447458bd5d72666a74624eddb0ef98bdcee61a0153aae0865ff"}, - "octo_fetch": {:hex, :octo_fetch, "0.3.0", "89ff501d2ac0448556ff1931634a538fe6d6cd358ba827ce1747e6a42a46efbf", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "c07e44f2214ab153743b7b3182f380798d0b294b1f283811c1e30cff64096d3d"}, + "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"}, "open_api_spex": {:hex, :open_api_spex, "3.18.0", "f9952b6bc8a1bf14168f3754981b7c8d72d015112bfedf2588471dd602e1e715", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "37849887ab67efab052376401fac28c0974b273ffaecd98f4532455ca0886464"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.2.1", "7b69ed4f40025c005de0b74fce8c0549625d59cb4df12d15c32fe6dc5076ff42", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "6d7a27b7cad2ad69a09cabf6670514cafcec717c8441beb5c96322bac3d05350"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, + "peep": {:git, "https://github.com/hauleth/peep.git", "d2e30ba21e8937bd00c10c19488cf1d111a3a39f", [branch: "ft/custom-prometheus-types"]}, "pg_types": {:hex, :pg_types, "0.4.0", "3ce365c92903c5bb59c0d56382d842c8c610c1b6f165e20c4b652c96fa7e9c14", [:rebar3], [], "hexpm", "b02efa785caececf9702c681c80a9ca12a39f9161a846ce17b01fb20aeeed7eb"}, "pgo": {:hex, :pgo, "0.14.0", "f53711d103d7565db6fc6061fcf4ff1007ab39892439be1bb02d9f686d7e6663", [:rebar3], [{:backoff, "~> 1.1.6", [hex: :backoff, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:pg_types, "~> 0.4.0", [hex: :pg_types, repo: "hexpm", optional: false]}], "hexpm", "71016c22599936e042dc0012ee4589d24c71427d266292f775ebf201d97df9c9"}, - "phoenix": {:hex, :phoenix, "1.7.10", "02189140a61b2ce85bb633a9b6fd02dff705a5f1596869547aeb2b2b95edd729", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "cf784932e010fd736d656d7fead6a584a4498efefe5b8227e9f383bf15bb79d0"}, + "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.3", "86e9878f833829c3f66da03d75254c155d91d72a201eb56ae83482328dc7ca93", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d36c401206f3011fefd63d04e8ef626ec8791975d9d107f9a0817d426f61ac07"}, - "phoenix_html": {:hex, :phoenix_html, "3.3.3", "380b8fb45912b5638d2f1d925a3771b4516b9a78587249cabe394e0a5d579dc9", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "923ebe6fec6e2e3b3e569dfbdc6560de932cd54b000ada0208b5f45024bdd76c"}, + "phoenix_html": {:hex, :phoenix_html, "3.3.4", "42a09fc443bbc1da37e372a5c8e6755d046f22b9b11343bf885067357da21cb3", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0249d3abec3714aff3415e7ee3d9786cb325be3151e6c4b3021502c585bf53fb"}, "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.7.2", "97cc4ff2dba1ebe504db72cb45098cb8e91f11160528b980bd282cc45c73b29c", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.18.3", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0e5fdf063c7a3b620c566a30fcf68b7ee02e5e46fe48ee46a6ec3ba382dc05b7"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.4.1", "2aff698f5e47369decde4357ba91fc9c37c6487a512b41732818f2204a8ef1d3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "9bffb834e7ddf08467fe54ae58b5785507aaba6255568ae22b4d46e2bb3615ab"}, "phoenix_live_view": {:hex, :phoenix_live_view, "0.18.18", "1f38fbd7c363723f19aad1a04b5490ff3a178e37daaf6999594d5f34796c47fc", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a5810d0472f3189ede6d2a95bda7f31c6113156b91784a3426cb0ab6a6d85214"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, - "phoenix_template": {:hex, :phoenix_template, "1.0.3", "32de561eefcefa951aead30a1f94f1b5f0379bc9e340bb5c667f65f1edfa4326", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "16f4b6588a4152f3cc057b9d0c0ba7e82ee23afa65543da535313ad8d25d8e2c"}, - "phoenix_view": {:hex, :phoenix_view, "2.0.3", "4d32c4817fce933693741deeb99ef1392619f942633dde834a5163124813aad3", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "cd34049af41be2c627df99cd4eaa71fc52a328c0c3d8e7d4aa28f880c30e7f64"}, - "plug": {:hex, :plug, "1.15.1", "b7efd81c1a1286f13efb3f769de343236bd8b7d23b4a9f40d3002fc39ad8f74c", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "459497bd94d041d98d948054ec6c0b76feacd28eec38b219ca04c0de13c79d30"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"}, - "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"}, + "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, + "phoenix_view": {:hex, :phoenix_view, "2.0.4", "b45c9d9cf15b3a1af5fb555c674b525391b6a1fe975f040fb4d913397b31abf4", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "4e992022ce14f31fe57335db27a28154afcc94e9983266835bb3040243eb620b"}, + "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, + "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "poolboy": {:git, "https://github.com/abc3/poolboy.git", "999ec7f5c7282d515020bb058b4832029d6d07bc", [tag: "v0.0.2"]}, "postgrex": {:hex, :postgrex, "0.17.3", "c92cda8de2033a7585dae8c61b1d420a1a1322421df84da9a82a6764580c503d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "946cf46935a4fdca7a81448be76ba3503cff082df42c6ec1ff16a4bdfbfb098d"}, - "prom_ex": {:hex, :prom_ex, "1.8.0", "662615e1d2f2ab3e0dc13a51c92ad0ccfcab24336a90cb9b114ee1bce9ef88aa", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.15", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.3", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "3eea763dfa941e25de50decbf17a6a94dbd2270e7b32f88279aa6e9bbb8e23e7"}, + "prom_ex": {:git, "https://github.com/hauleth/prom_ex.git", "c9b0793e7483d09a96252992debf30b7bb6b1216", [branch: "ft/add-peep-storage"]}, "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, "recon": {:hex, :recon, "2.5.4", "05dd52a119ee4059fa9daa1ab7ce81bc7a8161a2f12e9d42e9d551ffd2ba901c", [:mix, :rebar3], [], "hexpm", "e9ab01ac7fc8572e41eb59385efeb3fb0ff5bf02103816535bacaedf327d0263"}, "req": {:hex, :req, "0.3.12", "f84c2f9e7cc71c81d7cbeacf7c61e763e53ab5f3065703792a4ab264b4f22672", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "c91103d4d1c8edeba90c84e0ba223a59865b673eaab217bfd17da3aa54ab136c"}, "rustler": {:hex, :rustler, "0.29.1", "880f20ae3027bd7945def6cea767f5257bc926f33ff50c0d5d5a5315883c084d", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "109497d701861bfcd26eb8f5801fe327a8eef304f56a5b63ef61151ff44ac9b6"}, "sleeplocks": {:hex, :sleeplocks, "1.1.2", "d45aa1c5513da48c888715e3381211c859af34bee9b8290490e10c90bb6ff0ca", [:rebar3], [], "hexpm", "9fe5d048c5b781d6305c1a3a0f40bb3dfc06f49bf40571f3d2d0c57eaa7f59a5"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, "syn": {:hex, :syn, "3.3.0", "4684a909efdfea35ce75a9662fc523e4a8a4e8169a3df275e4de4fa63f99c486", [:rebar3], [], "hexpm", "e58ee447bc1094bdd21bf0acc102b1fbf99541a508cd48060bf783c245eaf7d6"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, - "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, + "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"}, "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.1.0", "4e15f6d7dbedb3a4e3aed2262b7e1407f166fcb9c30ca3f96635dfbbef99965c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0dd10e7fe8070095df063798f82709b0a1224c31b8baf6278b423898d591a069"}, - "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, + "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, "tesla": {:hex, :tesla, "1.8.0", "d511a4f5c5e42538d97eef7c40ec4f3e44effdc5068206f42ed859e09e51d1fd", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "10501f360cd926a309501287470372af1a6e1cbed0f43949203a4c13300bc79f"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, "unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, - "websock_adapter": {:hex, :websock_adapter, "0.5.5", "9dfeee8269b27e958a65b3e235b7e447769f66b5b5925385f5a569269164a210", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b977ba4a01918acbf77045ff88de7f6972c2a009213c515a445c48f224ffce9"}, + "websock_adapter": {:hex, :websock_adapter, "0.5.6", "0437fe56e093fd4ac422de33bf8fc89f7bc1416a3f2d732d8b2c8fd54792fe60", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "e04378d26b0af627817ae84c92083b7e97aca3121196679b73c73b99d0d133ea"}, } From 540bb81458c1e10d1c9b67c19e146c771849f539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Mon, 8 Jul 2024 12:00:19 +0200 Subject: [PATCH 13/97] chore: add comments about forks (#388) --- mix.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mix.exs b/mix.exs index 8d2f09e5..99a050ae 100644 --- a/mix.exs +++ b/mix.exs @@ -50,6 +50,7 @@ defmodule Supavisor.MixProject do {:phoenix_live_dashboard, "~> 0.7"}, {:telemetry_metrics, "~> 0.6"}, {:telemetry_poller, "~> 1.0"}, + # TODO: point it to Supabase fork of prom_ex when available {:peep, github: "hauleth/peep", branch: "ft/custom-prometheus-types", override: true}, {:jason, "~> 1.2"}, {:plug_cowboy, "~> 2.5"}, @@ -59,6 +60,7 @@ defmodule Supavisor.MixProject do {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, {:benchee, "~> 1.1.0", only: :dev}, + # TODO: point it to Supabase fork of prom_ex when available {:prom_ex, github: "hauleth/prom_ex", branch: "ft/add-peep-storage"}, {:open_api_spex, "~> 3.16"}, {:burrito, github: "burrito-elixir/burrito"}, From 36c8ac87a2f5d04a902a31184303ba34ae010e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Mon, 8 Jul 2024 12:01:01 +0200 Subject: [PATCH 14/97] fix: do not remove logging in dev environment (#387) --- config/dev.exs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index b941e7e8..2e30a3bc 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -58,11 +58,6 @@ config :supavisor, SupavisorWeb.Endpoint, ] ] -config :logger, - compile_time_purge_matching: [ - [level_lower_than: :info] - ] - # Configures Elixir's Logger config :logger, :console, format: "$time [$level] $message $metadata\n", From aaca1cfab289e71a34a9c814ea185cc7d7616543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Mon, 8 Jul 2024 12:01:57 +0200 Subject: [PATCH 15/97] chore: do not call `Access` twice (#370) --- lib/supavisor/manager.ex | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/supavisor/manager.ex b/lib/supavisor/manager.ex index 1c7c63d8..d7eee0d1 100644 --- a/lib/supavisor/manager.ex +++ b/lib/supavisor/manager.ex @@ -149,9 +149,13 @@ defmodule Supavisor.Manager do @spec check_parameter_status(map, map) :: :ok | {:error, String.t()} defp check_parameter_status(ps, def_ps) do - Enum.find_value(ps, :ok, fn {key, value} -> - if def_ps[key] && def_ps[key] != value do - {:error, "Parameter #{key} changed from #{def_ps[key]} to #{value}"} + Enum.find_value(ps, :ok, fn {key, new_value} -> + case def_ps do + %{^key => old_value} when old_value != new_value -> + {:error, "Parameter #{key} changed from #{old_value} to #{new_value}"} + + _ -> + nil end end) end From b1fa4494ef42ed27572c95cac62fc1d8328e4a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Mon, 8 Jul 2024 12:47:27 +0200 Subject: [PATCH 16/97] chore: update dependencies (#389) * chore: update Benchee * chore: unlock telemetry_metrics * chore: remove unused libraries from lockfile * chore: update Rustler and Rust dependencies * chore: update direct dependencies * chore: upddate Phoenix.LiveView --- lib/supavisor/handler_helpers.ex | 2 +- lib/supavisor/tenants/tenant.ex | 8 +- mix.exs | 11 +- mix.lock | 49 +++-- native/pgparser/Cargo.lock | 317 ++++++++++++------------------- native/pgparser/Cargo.toml | 2 +- 6 files changed, 154 insertions(+), 235 deletions(-) diff --git a/lib/supavisor/handler_helpers.ex b/lib/supavisor/handler_helpers.ex index 8dd9ec27..58c914a8 100644 --- a/lib/supavisor/handler_helpers.ex +++ b/lib/supavisor/handler_helpers.ex @@ -155,7 +155,7 @@ defmodule Supavisor.HandlerHelpers do @spec filter_cidrs(list(), :inet.ip_address() | any()) :: list() def filter_cidrs(allow_list, addr) when is_list(allow_list) and is_tuple(addr) do for range <- allow_list, - range |> InetCidr.parse() |> InetCidr.contains?(addr) do + range |> InetCidr.parse_cidr!() |> InetCidr.contains?(addr) do range end end diff --git a/lib/supavisor/tenants/tenant.ex b/lib/supavisor/tenants/tenant.ex index cf1941e6..7864f2bd 100644 --- a/lib/supavisor/tenants/tenant.ex +++ b/lib/supavisor/tenants/tenant.ex @@ -121,12 +121,6 @@ defmodule Supavisor.Tenants.Tenant do end defp valid_range?(range) do - try do - InetCidr.parse(range) - true - rescue - _e -> - false - end + match?({:ok, _}, InetCidr.parse_cidr(range)) end end diff --git a/mix.exs b/mix.exs index 99a050ae..b416a9bd 100644 --- a/mix.exs +++ b/mix.exs @@ -46,20 +46,19 @@ defmodule Supavisor.MixProject do {:phoenix_html, "~> 3.0"}, {:phoenix_view, "~> 2.0.2"}, {:phoenix_live_reload, "~> 1.2", only: :dev}, - {:phoenix_live_view, "~> 0.18.18"}, + {:phoenix_live_view, "~> 0.20.0"}, {:phoenix_live_dashboard, "~> 0.7"}, - {:telemetry_metrics, "~> 0.6"}, {:telemetry_poller, "~> 1.0"}, # TODO: point it to Supabase fork of prom_ex when available {:peep, github: "hauleth/peep", branch: "ft/custom-prometheus-types", override: true}, {:jason, "~> 1.2"}, {:plug_cowboy, "~> 2.5"}, - {:joken, "~> 2.5.0"}, - {:cloak_ecto, "~> 1.2.0"}, + {:joken, "~> 2.6.0"}, + {:cloak_ecto, "~> 1.3.0"}, {:meck, "~> 0.9.2", only: [:dev, :test]}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, - {:benchee, "~> 1.1.0", only: :dev}, + {:benchee, "~> 1.3", only: :dev}, # TODO: point it to Supabase fork of prom_ex when available {:prom_ex, github: "hauleth/prom_ex", branch: "ft/add-peep-storage"}, {:open_api_spex, "~> 3.16"}, @@ -77,7 +76,7 @@ defmodule Supavisor.MixProject do {:poolboy, git: "https://github.com/abc3/poolboy.git", tag: "v0.0.2"}, {:syn, "~> 3.3"}, {:pgo, "~> 0.13"}, - {:rustler, "~> 0.29.1"}, + {:rustler, "~> 0.33.0"}, {:ranch, "~> 2.0", override: true} ] end diff --git a/mix.lock b/mix.lock index a4125f3e..5a43cde9 100644 --- a/mix.lock +++ b/mix.lock @@ -1,35 +1,35 @@ %{ "artificery": {:hex, :artificery, "0.4.3", "0bc4260f988dcb9dda4b23f9fc3c6c8b99a6220a331534fdf5bf2fd0d4333b02", [:mix], [], "hexpm", "12e95333a30e20884e937abdbefa3e7f5e05609c2ba8cf37b33f000b9ffc0504"}, "backoff": {:hex, :backoff, "1.1.6", "83b72ed2108ba1ee8f7d1c22e0b4a00cfe3593a67dbc792799e8cce9f42f796b", [:rebar3], [], "hexpm", "cf0cfff8995fb20562f822e5cc47d8ccf664c5ecdc26a684cbe85c225f9d7c39"}, - "benchee": {:hex, :benchee, "1.1.0", "f3a43817209a92a1fade36ef36b86e1052627fd8934a8b937ac9ab3a76c43062", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}], "hexpm", "7da57d545003165a012b587077f6ba90b89210fd88074ce3c60ce239eb5e6d93"}, + "benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"}, "bertex": {:hex, :bertex, "1.3.0", "0ad0df9159b5110d9d2b6654f72fbf42a54884ef43b6b651e6224c0af30ba3cb", [:mix], [], "hexpm", "0a5d5e478bb5764b7b7bae37cae1ca491200e58b089df121a2fe1c223d8ee57a"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, - "burrito": {:git, "https://github.com/burrito-elixir/burrito.git", "a60c6ab21156fc4c788907d33bfd0c546a022272", []}, + "burrito": {:git, "https://github.com/burrito-elixir/burrito.git", "efd6a329f26e4039e7d46d00e571dc1df5b69262", []}, "cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"}, "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, - "cloak": {:hex, :cloak, "1.1.2", "7e0006c2b0b98d976d4f559080fabefd81f0e0a50a3c4b621f85ceeb563e80bb", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "940d5ac4fcd51b252930fd112e319ea5ae6ab540b722f3ca60a85666759b9585"}, - "cloak_ecto": {:hex, :cloak_ecto, "1.2.0", "e86a3df3bf0dc8980f70406bcb0af2858bac247d55494d40bc58a152590bd402", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "8bcc677185c813fe64b786618bd6689b1707b35cd95acaae0834557b15a0c62f"}, + "cloak": {:hex, :cloak, "1.1.4", "aba387b22ea4d80d92d38ab1890cc528b06e0e7ef2a4581d71c3fdad59e997e7", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "92b20527b9aba3d939fab0dd32ce592ff86361547cfdc87d74edce6f980eb3d7"}, + "cloak_ecto": {:hex, :cloak_ecto, "1.3.0", "0de127c857d7452ba3c3367f53fb814b0410ff9c680a8d20fbe8b9a3c57a1118", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "314beb0c123b8a800418ca1d51065b27ba3b15f085977e65c0f7b2adab2de1cc"}, "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, - "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"}, + "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "distillery": {:hex, :distillery, "2.1.1", "f9332afc2eec8a1a2b86f22429e068ef35f84a93ea1718265e740d90dd367814", [:mix], [{:artificery, "~> 0.2", [hex: :artificery, repo: "hexpm", optional: false]}], "hexpm", "bbc7008b0161a6f130d8d903b5b3232351fccc9c31a991f8fcbf2a12ace22995"}, - "ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"}, - "ecto_sql": {:hex, :ecto_sql, "3.10.2", "6b98b46534b5c2f8b8b5f03f126e75e2a73c64f3c071149d32987a5378b0fdbd", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "68c018debca57cb9235e3889affdaec7a10616a4e3a80c99fa1d01fdafaa9007"}, + "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, + "ecto_sql": {:hex, :ecto_sql, "3.11.3", "4eb7348ff8101fbc4e6bbc5a4404a24fecbe73a3372d16569526b0cf34ebc195", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e5f36e3d736b99c7fee3e631333b8394ade4bafe9d96d35669fca2d81c2be928"}, "eflambe": {:hex, :eflambe, "0.3.1", "ef0a35084fad1f50744496730a9662782c0a9ebf449d3e03143e23295c5926ea", [:rebar3], [{:meck, "0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "58d5997be606d4e269e9e9705338e055281fdf3e4935cc902c8908e9e4516c5f"}, "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, - "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, + "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"}, - "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"}, + "inet_cidr": {:hex, :inet_cidr, "1.0.8", "d26bb7bdbdf21ae401ead2092bf2bb4bf57fe44a62f5eaa5025280720ace8a40", [:mix], [], "hexpm", "d5b26da66603bb56c933c65214c72152f0de9a6ea53618b56d63302a68f6a90e"}, "jason": {:hex, :jason, "1.4.3", "d3f984eeb96fe53b85d20e0b049f03e57d075b5acda3ac8d465c969a2536c17b", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "9a90e868927f7c777689baa16d86f4d0e086d968db5c05d917ccff6d443e58a3"}, - "joken": {:hex, :joken, "2.5.0", "09be497d804b8115eb6f07615cef2e60c2a1008fb89dc0aef0d4c4b4609b99aa", [:mix], [{:jose, "~> 1.11.2", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "22b25c89617c5ed8ca7b31026340a25ea0f9ca7160f9706b79be9ed81fdf74e7"}, - "jose": {:hex, :jose, "1.11.6", "613fda82552128aa6fb804682e3a616f4bc15565a048dabd05b1ebd5827ed965", [:mix, :rebar3], [], "hexpm", "6275cb75504f9c1e60eeacb771adfeee4905a9e182103aa59b53fed651ff9738"}, + "joken": {:hex, :joken, "2.6.1", "2ca3d8d7f83bf7196296a3d9b2ecda421a404634bfc618159981a960020480a1", [:mix], [{:jose, "~> 1.11.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "ab26122c400b3d254ce7d86ed066d6afad27e70416df947cdcb01e13a7382e68"}, + "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, "jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"}, "libcluster": {:hex, :libcluster, "3.3.3", "a4f17721a19004cfc4467268e17cff8b1f951befe428975dd4f6f7b84d927fe0", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "7c0a2275a0bb83c07acd17dab3c3bfb4897b145106750eeccc62d302e3bdfee5"}, "logflare_api_client": {:hex, :logflare_api_client, "0.3.5", "c427ebf65a8402d68b056d4a5ef3e1eb3b90c0ad1d0de97d1fe23807e0c1b113", [:mix], [{:bertex, "~> 1.3", [hex: :bertex, repo: "hexpm", optional: false]}, {:finch, "~> 0.10", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: false]}, {:tesla, "~> 1.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "16d29abcb80c4f72745cdf943379da02a201504813c3aa12b4d4acb0302b7723"}, @@ -42,18 +42,18 @@ "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "observer_cli": {:hex, :observer_cli, "1.7.4", "3c1bfb6d91bf68f6a3d15f46ae20da0f7740d363ee5bc041191ce8722a6c4fae", [:mix, :rebar3], [{:recon, "~> 2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "50de6d95d814f447458bd5d72666a74624eddb0ef98bdcee61a0153aae0865ff"}, "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"}, - "open_api_spex": {:hex, :open_api_spex, "3.18.0", "f9952b6bc8a1bf14168f3754981b7c8d72d015112bfedf2588471dd602e1e715", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "37849887ab67efab052376401fac28c0974b273ffaecd98f4532455ca0886464"}, - "opentelemetry_api": {:hex, :opentelemetry_api, "1.2.1", "7b69ed4f40025c005de0b74fce8c0549625d59cb4df12d15c32fe6dc5076ff42", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "6d7a27b7cad2ad69a09cabf6670514cafcec717c8441beb5c96322bac3d05350"}, + "open_api_spex": {:hex, :open_api_spex, "3.19.1", "65ccb5d06e3d664d1eec7c5ea2af2289bd2f37897094a74d7219fb03fc2b5994", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "392895827ce2984a3459c91a484e70708132d8c2c6c5363972b4b91d6bbac3dd"}, + "opentelemetry_api": {:hex, :opentelemetry_api, "1.3.0", "03e2177f28dd8d11aaa88e8522c81c2f6a788170fe52f7a65262340961e663f9", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "b9e5ff775fd064fa098dba3c398490b77649a352b40b0b730a6b7dc0bdd68858"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, "peep": {:git, "https://github.com/hauleth/peep.git", "d2e30ba21e8937bd00c10c19488cf1d111a3a39f", [branch: "ft/custom-prometheus-types"]}, "pg_types": {:hex, :pg_types, "0.4.0", "3ce365c92903c5bb59c0d56382d842c8c610c1b6f165e20c4b652c96fa7e9c14", [:rebar3], [], "hexpm", "b02efa785caececf9702c681c80a9ca12a39f9161a846ce17b01fb20aeeed7eb"}, "pgo": {:hex, :pgo, "0.14.0", "f53711d103d7565db6fc6061fcf4ff1007ab39892439be1bb02d9f686d7e6663", [:rebar3], [{:backoff, "~> 1.1.6", [hex: :backoff, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:pg_types, "~> 0.4.0", [hex: :pg_types, repo: "hexpm", optional: false]}], "hexpm", "71016c22599936e042dc0012ee4589d24c71427d266292f775ebf201d97df9c9"}, "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"}, - "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.3", "86e9878f833829c3f66da03d75254c155d91d72a201eb56ae83482328dc7ca93", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d36c401206f3011fefd63d04e8ef626ec8791975d9d107f9a0817d426f61ac07"}, + "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.2", "3b83b24ab5a2eb071a20372f740d7118767c272db386831b2e77638c4dcc606d", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "3f94d025f59de86be00f5f8c5dd7b5965a3298458d21ab1c328488be3b5fcd59"}, "phoenix_html": {:hex, :phoenix_html, "3.3.4", "42a09fc443bbc1da37e372a5c8e6755d046f22b9b11343bf885067357da21cb3", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0249d3abec3714aff3415e7ee3d9786cb325be3151e6c4b3021502c585bf53fb"}, - "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.7.2", "97cc4ff2dba1ebe504db72cb45098cb8e91f11160528b980bd282cc45c73b29c", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.18.3", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0e5fdf063c7a3b620c566a30fcf68b7ee02e5e46fe48ee46a6ec3ba382dc05b7"}, - "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.4.1", "2aff698f5e47369decde4357ba91fc9c37c6487a512b41732818f2204a8ef1d3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "9bffb834e7ddf08467fe54ae58b5785507aaba6255568ae22b4d46e2bb3615ab"}, - "phoenix_live_view": {:hex, :phoenix_live_view, "0.18.18", "1f38fbd7c363723f19aad1a04b5490ff3a178e37daaf6999594d5f34796c47fc", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a5810d0472f3189ede6d2a95bda7f31c6113156b91784a3426cb0ab6a6d85214"}, + "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.4", "4508e481f791ce62ec6a096e13b061387158cbeefacca68c6c1928e1305e23ed", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "2984aae96994fbc5c61795a73b8fb58153b41ff934019cfb522343d2d3817d59"}, + "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.5.3", "f2161c207fda0e4fb55165f650f7f8db23f02b29e3bff00ff7ef161d6ac1f09d", [:mix], [{:file_system, "~> 0.3 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "b4ec9cd73cb01ff1bd1cac92e045d13e7030330b74164297d1aee3907b54803c"}, + "phoenix_live_view": {:hex, :phoenix_live_view, "0.20.17", "f396bbdaf4ba227b82251eb75ac0afa6b3da5e509bc0d030206374237dfc9450", [:mix], [{:floki, "~> 0.36", [hex: :floki, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a61d741ffb78c85fdbca0de084da6a48f8ceb5261a79165b5a0b59e5f65ce98b"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, "phoenix_view": {:hex, :phoenix_view, "2.0.4", "b45c9d9cf15b3a1af5fb555c674b525391b6a1fe975f040fb4d913397b31abf4", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "4e992022ce14f31fe57335db27a28154afcc94e9983266835bb3040243eb620b"}, @@ -61,21 +61,20 @@ "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "poolboy": {:git, "https://github.com/abc3/poolboy.git", "999ec7f5c7282d515020bb058b4832029d6d07bc", [tag: "v0.0.2"]}, - "postgrex": {:hex, :postgrex, "0.17.3", "c92cda8de2033a7585dae8c61b1d420a1a1322421df84da9a82a6764580c503d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "946cf46935a4fdca7a81448be76ba3503cff082df42c6ec1ff16a4bdfbfb098d"}, + "postgrex": {:hex, :postgrex, "0.18.0", "f34664101eaca11ff24481ed4c378492fed2ff416cd9b06c399e90f321867d7e", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a042989ba1bc1cca7383ebb9e461398e3f89f868c92ce6671feb7ef132a252d1"}, "prom_ex": {:git, "https://github.com/hauleth/prom_ex.git", "c9b0793e7483d09a96252992debf30b7bb6b1216", [branch: "ft/add-peep-storage"]}, "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, - "recon": {:hex, :recon, "2.5.4", "05dd52a119ee4059fa9daa1ab7ce81bc7a8161a2f12e9d42e9d551ffd2ba901c", [:mix, :rebar3], [], "hexpm", "e9ab01ac7fc8572e41eb59385efeb3fb0ff5bf02103816535bacaedf327d0263"}, - "req": {:hex, :req, "0.3.12", "f84c2f9e7cc71c81d7cbeacf7c61e763e53ab5f3065703792a4ab264b4f22672", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "c91103d4d1c8edeba90c84e0ba223a59865b673eaab217bfd17da3aa54ab136c"}, - "rustler": {:hex, :rustler, "0.29.1", "880f20ae3027bd7945def6cea767f5257bc926f33ff50c0d5d5a5315883c084d", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "109497d701861bfcd26eb8f5801fe327a8eef304f56a5b63ef61151ff44ac9b6"}, - "sleeplocks": {:hex, :sleeplocks, "1.1.2", "d45aa1c5513da48c888715e3381211c859af34bee9b8290490e10c90bb6ff0ca", [:rebar3], [], "hexpm", "9fe5d048c5b781d6305c1a3a0f40bb3dfc06f49bf40571f3d2d0c57eaa7f59a5"}, + "recon": {:hex, :recon, "2.5.5", "c108a4c406fa301a529151a3bb53158cadc4064ec0c5f99b03ddb8c0e4281bdf", [:mix, :rebar3], [], "hexpm", "632a6f447df7ccc1a4a10bdcfce71514412b16660fe59deca0fcf0aa3c054404"}, + "req": {:hex, :req, "0.4.0", "1c759054dd64ef1b1a0e475c2d2543250d18f08395d3174c371b7746984579ce", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "f53eadc32ebefd3e5d50390356ec3a59ed2b8513f7da8c6c3f2e14040e9fe989"}, + "rustler": {:hex, :rustler, "0.33.0", "4a5b0a7a7b0b51549bea49947beff6fae9bc5d5326104dcd4531261e876b5619", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "7c4752728fee59a815ffd20c3429c55b644041f25129b29cdeb5c470b80ec5fd"}, + "sleeplocks": {:hex, :sleeplocks, "1.1.3", "96a86460cc33b435c7310dbd27ec82ca2c1f24ae38e34f8edde97f756503441a", [:rebar3], [], "hexpm", "d3b3958552e6eb16f463921e70ae7c767519ef8f5be46d7696cc1ed649421321"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, "syn": {:hex, :syn, "3.3.0", "4684a909efdfea35ce75a9662fc523e4a8a4e8169a3df275e4de4fa63f99c486", [:rebar3], [], "hexpm", "e58ee447bc1094bdd21bf0acc102b1fbf99541a508cd48060bf783c245eaf7d6"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, - "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"}, - "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.1.0", "4e15f6d7dbedb3a4e3aed2262b7e1407f166fcb9c30ca3f96635dfbbef99965c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0dd10e7fe8070095df063798f82709b0a1224c31b8baf6278b423898d591a069"}, + "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"}, "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, - "tesla": {:hex, :tesla, "1.8.0", "d511a4f5c5e42538d97eef7c40ec4f3e44effdc5068206f42ed859e09e51d1fd", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "10501f360cd926a309501287470372af1a6e1cbed0f43949203a4c13300bc79f"}, + "tesla": {:hex, :tesla, "1.11.1", "902ec0cd9fb06ba534be765f0eb78acd9d0ef70118230dc3a73fdc9afc91d036", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "c02d7dd149633c55c40adfaad6c3ce2615cfc89258b67a7f428c14bb835c398c"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, "unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"}, diff --git a/native/pgparser/Cargo.lock b/native/pgparser/Cargo.lock index 7233ff8a..9e558302 100644 --- a/native/pgparser/Cargo.lock +++ b/native/pgparser/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "anyhow" -version = "1.0.79" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "bindgen" @@ -23,7 +23,7 @@ version = "0.66.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" dependencies = [ - "bitflags 2.4.1", + "bitflags", "cexpr", "clang-sys", "lazy_static", @@ -36,36 +36,27 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.48", + "syn 2.0.69", "which", ] [[package]] name = "bitflags" -version = "1.3.2" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] +checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437" [[package]] name = "cexpr" @@ -84,9 +75,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clang-sys" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -104,9 +95,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "equivalent" @@ -116,19 +107,19 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] name = "fastrand" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fixedbitset" @@ -150,9 +141,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "heck" @@ -160,20 +151,26 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "home" version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.52.0", + "windows-sys", ] [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown", @@ -190,15 +187,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lazycell" @@ -208,37 +205,37 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.152" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" -version = "0.8.1" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" +checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-targets", ] [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "log" -version = "0.4.20" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "minimal-lexical" @@ -276,9 +273,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", "indexmap", @@ -312,19 +309,19 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.16" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.48", + "syn 2.0.69", ] [[package]] name = "proc-macro2" -version = "1.0.76" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -348,7 +345,7 @@ dependencies = [ "bytes", "cfg-if", "cmake", - "heck", + "heck 0.4.1", "itertools", "lazy_static", "log", @@ -386,27 +383,18 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "regex" -version = "1.10.2" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", @@ -416,9 +404,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", @@ -427,9 +415,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "rustc-hash" @@ -439,45 +427,44 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.4.1", + "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] name = "rustler" -version = "0.30.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4b4fea69e23de68c42c06769d6624d2d018da550c17244dd4b691f90ced4a7e" +checksum = "45d51ae0239c57c3a3e603dd855ace6795078ef33c95c85d397a100ac62ed352" dependencies = [ - "lazy_static", "rustler_codegen", "rustler_sys", ] [[package]] name = "rustler_codegen" -version = "0.30.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406061bd07aaf052c344257afed4988c5ec8efe4d2352b4c2cf27ea7c8575b12" +checksum = "27061f1a2150ad64717dca73902678c124b0619b0d06563294df265bc84759e1" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.69", ] [[package]] name = "rustler_sys" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a7c0740e5322b64e2b952d8f0edce5f90fcf6f6fe74cca3f6e78eb3de5ea858" +checksum = "2062df0445156ae93cf695ef38c00683848d956b30507592143c01fe8fb52fda" dependencies = [ "regex", "unreachable", @@ -485,35 +472,35 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.69", ] [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", @@ -539,9 +526,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "201fcda3845c23e8212cd466bfebf0bd20694490fc0356ae8e428e0824a915a6" dependencies = [ "proc-macro2", "quote", @@ -550,35 +537,34 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", "rustix", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.69", ] [[package]] @@ -614,134 +600,75 @@ dependencies = [ "rustix", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" -dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] -name = "windows_i686_msvc" -version = "0.48.5" +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/native/pgparser/Cargo.toml b/native/pgparser/Cargo.toml index a0e0c61e..6c598da2 100644 --- a/native/pgparser/Cargo.toml +++ b/native/pgparser/Cargo.toml @@ -9,5 +9,5 @@ path = "src/lib.rs" crate-type = ["cdylib"] [dependencies] -rustler = "0.30.0" +rustler = "0.33.0" pg_query = "5.1.0" From 6952f8b8b93688d97f584b1bdc34bd743e350ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Mon, 8 Jul 2024 19:25:11 +0200 Subject: [PATCH 17/97] chore: remove common_test from required applications (#391) --- mix.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/mix.exs b/mix.exs index b416a9bd..f7ce8382 100644 --- a/mix.exs +++ b/mix.exs @@ -26,7 +26,6 @@ defmodule Supavisor.MixProject do ] end - defp extra_applications(:test), do: [:common_test] defp extra_applications(:dev), do: [:wx, :observer] defp extra_applications(_), do: [] From 699f9ff8b55ed6a2642bfffe3682bccf5d666b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jan=20Niemier?= Date: Tue, 9 Jul 2024 09:54:38 +0200 Subject: [PATCH 18/97] ft: remove all HTML-related functionalities (#392) Supavisor is not using any HTML views, it do not offer any HTML-based UI, etc. so these functionalities aren't needed for anything. --- lib/supavisor_web.ex | 49 ++----------------- .../templates/layout/app.html.heex | 5 -- .../templates/layout/live.html.heex | 11 ----- .../templates/layout/root.html.heex | 13 ----- lib/supavisor_web/views/error_helpers.ex | 14 ------ lib/supavisor_web/views/layout_view.ex | 7 --- mix.exs | 1 - test/supavisor_web/views/layout_view_test.exs | 8 --- 8 files changed, 3 insertions(+), 105 deletions(-) delete mode 100644 lib/supavisor_web/templates/layout/app.html.heex delete mode 100644 lib/supavisor_web/templates/layout/live.html.heex delete mode 100644 lib/supavisor_web/templates/layout/root.html.heex delete mode 100644 lib/supavisor_web/views/layout_view.ex delete mode 100644 test/supavisor_web/views/layout_view_test.exs diff --git a/lib/supavisor_web.ex b/lib/supavisor_web.ex index 3db43269..66000550 100644 --- a/lib/supavisor_web.ex +++ b/lib/supavisor_web.ex @@ -32,37 +32,10 @@ defmodule SupavisorWeb do root: "lib/supavisor_web/templates", namespace: SupavisorWeb - # Import convenience functions from controllers - import Phoenix.Controller, - only: [get_flash: 1, get_flash: 2, view_module: 1, view_template: 1] - - # Include shared imports and aliases for views - unquote(view_helpers()) - end - end - - def live_view do - quote do - use Phoenix.LiveView, - layout: {SupavisorWeb.LayoutView, "live.html"} - - unquote(view_helpers()) - end - end - - def live_component do - quote do - use Phoenix.LiveComponent - - unquote(view_helpers()) - end - end - - def component do - quote do - use Phoenix.Component + # Import basic rendering functionality (render, render_layout, etc) + import Phoenix.View - unquote(view_helpers()) + import SupavisorWeb.ErrorHelpers end end @@ -82,22 +55,6 @@ defmodule SupavisorWeb do end end - defp view_helpers do - quote do - # Use all HTML functionality (forms, tags, etc) - use Phoenix.HTML - - # Import LiveView and .heex helpers (live_render, live_patch, <.form>, etc) - import Phoenix.Component - - # Import basic rendering functionality (render, render_layout, etc) - import Phoenix.View - - import SupavisorWeb.ErrorHelpers - alias SupavisorWeb.Router.Helpers, as: Routes - end - end - @doc """ When used, dispatch to the appropriate controller/view/etc. """ diff --git a/lib/supavisor_web/templates/layout/app.html.heex b/lib/supavisor_web/templates/layout/app.html.heex deleted file mode 100644 index 169aed95..00000000 --- a/lib/supavisor_web/templates/layout/app.html.heex +++ /dev/null @@ -1,5 +0,0 @@ -